Skip to content
GitLab
Commit f0b931c9 authored by Diego Molteni's avatar Diego Molteni
Browse files

rebased with SLB changes

parent cc34ee4a
Pipeline #26731 passed with stages
in 8 minutes and 44 seconds
Expand all Hide whitespace changes
Inline Side-by-side
.dockerignore 0 → 100644
View file @ f0b931c9
node_modules
.gitignore
View file @ f0b931c9
......@@ -27,4 +27,7 @@ keys
.env
# newman junit output
newman
\ No newline at end of file
newman
# backup files
*.bak
\ No newline at end of file
scripts/genchangelog.sh → devops/scripts/genchangelog.sh
View file @ f0b931c9
......@@ -2,7 +2,7 @@
# This script generate a detailed changelog for the released changes to the seismic store service
logname='../CHANGELOG.md'
logname='CHANGELOG.md'
printf "%s\n" "# Seismic Store Change Log" > $logname
......
docker/runtime.Dockerfile
View file @ f0b931c9
......@@ -35,4 +35,4 @@ FROM node:${docker_node_image_version} as release
COPY --from=runtime-builder /service/artifact /seistore-service
WORKDIR /seistore-service
RUN npm install --production
ENTRYPOINT ["node", "./dist/server/server-start.js"]
\ No newline at end of file
ENTRYPOINT ["node", "./dist/server/server-start.js"]
docs/api/openapi.yaml
View file @ f0b931c9
This diff is collapsed.
docs/api/seismic_dms_seistore_openapi.yaml
View file @ f0b931c9
This diff is collapsed.
npm-shrinkwrap.json
View file @ f0b931c9
This diff is collapsed.
package.json
View file @ f0b931c9
......@@ -47,43 +47,46 @@
"@azure/keyvault-secrets": "^4.0.4",
"@azure/storage-blob": "^12.1.2",
"@cloudant/cloudant": "^4.2.4",
"@google-cloud/datastore": "5.0.5",
"@google-cloud/logging": "^7.3.0",
"@google-cloud/logging-winston": "3.0.5",
"@google-cloud/storage": "4.3.1",
"@google-cloud/trace-agent": "^5.1.0",
"@opencensus/core": "0.0.19",
"@opencensus/exporter-stackdriver": "0.0.19",
"@google-cloud/datastore": "6.3.1",
"@google-cloud/logging": "^9.0.0",
"@google-cloud/logging-winston": "4.0.2",
"@google-cloud/storage": "5.7.0",
"@google-cloud/trace-agent": "^5.1.1",
"@opencensus/core": "0.0.22",
"@opencensus/exporter-stackdriver": "0.0.22",
"applicationinsights": "^1.8.2",
"applicationinsights-native-metrics": "0.0.5",
"aws-sdk": "^2.739.0",
"body-parser": "1.19.0",
"bull": "^3.20.0",
"cors": "^2.8.5",
"dotenv": "^8.2.0",
"express": "4.17.1",
"extend": "^3.0.2",
"jsonwebtoken": "8.5.1",
"jwtproxy": "^1.6.3",
"jwtproxy": "^1.6.8",
"keycloak-admin": "1.13.0",
"lodash": "^4.17.20",
"log4js": "^6.3.0",
"minimist": "^1.2.5",
"mkdirp": "^1.0.4",
"node-pre-gyp": "^0.14.0",
"node-pre-gyp": "^0.17.0",
"redis": "^3.0.2",
"redlock": "^4.1.0",
"redlock-async": "^3.1.2-fix.2",
"replace-in-file": "^5.0.2",
"replace-in-file": "^6.1.0",
"request": "2.88.2",
"request-promise": "4.2.6",
"typescript": "^3.8.3",
"uuid": "^8.3.2",
"winston": "3.3.3",
"xss-filters": "1.2.7",
"yamljs": "0.3.0",
"yargs": "^15.3.1",
"yargs": "^16.2.0",
"yargs-parser": "^18.1.3"
},
"devDependencies": {
"@types/bull": "^3.14.4",
"@types/chai": "4.2.9",
"@types/cors": "^2.8.6",
"@types/express": "4.17.2",
......@@ -95,6 +98,7 @@
"@types/request": "2.48.4",
"@types/request-promise": "4.1.45",
"@types/sinon": "^7.5.1",
"@types/uuid": "^8.3.0",
"@types/xss-filters": "0.0.27",
"@types/yamljs": "0.2.30",
"async": "^3.2.0",
......
provider/gcp/cloudbuild/Dockerfile deleted 100644 → 0
View file @ cc34ee4a
# ============================================================================
# Copyright 2017-2019, Schlumberger
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ============================================================================
# [seistore runtime image]
ARG docker_node_image_version=12.18.2
# build the service (require builder image)
FROM ubuntu:bionic as runtime-builder
# nodejs version
ARG nodesecure_version=10
# update package list and install required packages
RUN apt-get update
RUN apt-get install -y curl
RUN apt-get install -y gnupg
RUN apt-get install -y git
# setup node from secure package
RUN curl -sL https://deb.nodesource.com/setup_${nodesecure_version}.x -o tmp/nodesource_setup.sh
RUN bash tmp/nodesource_setup.sh
RUN rm -f tmp/nodesource_setup.sh
# install nodejs and typescript globally
RUN apt-get install -y nodejs
RUN npm install -g typescript
ADD ./ /service
WORKDIR /service
RUN npm run clean && rm -rf node_modules && rm -rf artifact && mkdir artifact
RUN npm install
RUN npm run build
RUN cp -r package.json npm-shrinkwrap.json dist artifact
# Create the runtime image (require base image)
FROM node:${docker_node_image_version} as release
COPY --from=runtime-builder /service/artifact /seistore-service
WORKDIR /seistore-service
RUN npm install --production
ENTRYPOINT ["node", "./dist/server/server-start.js"]
\ No newline at end of file
provider/gcp/cloudbuild/cloudbuild.yaml deleted 100644 → 0
View file @ cc34ee4a
steps:
- name: 'gcr.io/cloud-builders/docker'
args: [
'build',
'-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}',
'-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:latest',
'-f', 'provider/${_PROVIDER_NAME}/cloudbuild/Dockerfile',
'.'
]
images:
- 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}'
src/auth/auth.ts
View file @ f0b931c9
......@@ -19,23 +19,44 @@ import { DESCompliance, DESUtils } from '../dataecosystem';
import { ImpTokenDAO } from '../services/imptoken';
import { AppsDAO } from '../services/svcapp/dao';
import { TenantModel } from '../services/tenant';
import { Error, Utils } from '../shared';
import { Cache, Error, Utils } from '../shared';
import { AuthGroups } from './groups';
export class Auth {
private static _cache: Cache<boolean>;
private static _cacheItemTTL = 60; // cache item expire after
public static async isUserRegistered(userToken: string, esd: string, appkey: string) {
await AuthGroups.getUserGroups(userToken, esd, appkey);
}
public static async isUserAuthorized(
authToken: string, authGroupsName: string[],
authToken: string, authGroupEmails: string[],
esd: string, appkey: string, mustThrow = true): Promise<boolean> {
const isAuthorized = await AuthGroups.hasOneInGroups(authToken, authGroupsName, esd, appkey);
if (!this._cache) {
this._cache = new Cache<boolean>({
ADDRESS: Config.DES_REDIS_INSTANCE_ADDRESS,
PORT: Config.DES_REDIS_INSTANCE_PORT,
KEY: Config.DES_REDIS_INSTANCE_KEY,
DISABLE_TLS: Config.DES_REDIS_INSTANCE_TLS_DISABLE,
}, 'auth')
}
const cacheKey = Utils.getEmailFromTokenPayload(authToken) + ',' + authGroupEmails.sort().join(',');
let isAuthorized = await this._cache.get(cacheKey);
if (isAuthorized === undefined) { // key not exist in cache -> canll entitlement
isAuthorized = await AuthGroups.isMemberOfAtleastOneGroup(authToken, authGroupEmails, esd, appkey);
await this._cache.set(cacheKey, isAuthorized, this._cacheItemTTL);
}
if (mustThrow && !isAuthorized) {
throw (Error.make(Error.Status.PERMISSION_DENIED,
'User not authorized to perform this operation'));
}
return isAuthorized;
}
......@@ -77,7 +98,7 @@ export class Auth {
}
public static async isLegalTagValid(
userToken: string,ltag: string, esd: string, appkey: string, mustThrow: boolean = true): Promise<boolean> {
userToken: string, ltag: string, esd: string, appkey: string, mustThrow: boolean = true): Promise<boolean> {
const entitlementTenant = DESUtils.getDataPartitionID(esd);
const isValid = await DESCompliance.isLegaTagValid(userToken, ltag, entitlementTenant, appkey);
if (mustThrow && !isValid) {
......
src/auth/groups.ts
View file @ f0b931c9
......@@ -69,9 +69,10 @@ export class AuthGroups {
}
public static async addUserToGroup(
userToken: string, group: string, userEmail: string, esd: string, appkey: string, role = 'MEMBER') {
userToken: string, group: string, userEmail: string,
esd: string, appkey: string, role = 'MEMBER', checkConsistencyForCreateGroup = false) {
await DESEntitlement.addUserToGroup(userToken, group, DESUtils.getDataPartitionID(esd), userEmail,
role, appkey);
role, appkey, checkConsistencyForCreateGroup);
}
public static async removeUserFromGroup(
......@@ -82,21 +83,21 @@ export class AuthGroups {
public static async listUsersInGroup(userToken: string, group: string, esd: string, appkey: string):
Promise<IDESEntitlementMemberModel[]> {
const entitlementTenant = DESUtils.getDataPartitionID(esd);
return (await DESEntitlement.listUsersInGroup(userToken, group, entitlementTenant, appkey)).members;
const entitlementTenant = DESUtils.getDataPartitionID(esd);
return (await DESEntitlement.listUsersInGroup(userToken, group, entitlementTenant, appkey)).members;
}
public static async getUserGroups(
userToken: string, esd: string, appkey: string): Promise<IDESEntitlementGroupModel[]> {
const entitlementTenant = DESUtils.getDataPartitionID(esd);
return await DESEntitlement.getUserGroups(userToken, entitlementTenant, appkey);
const entitlementTenant = DESUtils.getDataPartitionID(esd);
return await DESEntitlement.getUserGroups(userToken, entitlementTenant, appkey);
}
public static async hasOneInGroups(
userToken: string, groupsRef: string[], esd: string, appkey: string): Promise<boolean> {
public static async isMemberOfAtleastOneGroup(
userToken: string, groupEmails: string[], esd: string, appkey: string): Promise<boolean> {
const entitlementTenant = DESUtils.getDataPartitionID(esd);
const groups = await DESEntitlement.getUserGroups(userToken, entitlementTenant, appkey);
return groupsRef.some((groupsRefItem) => (groups.map((group) => group.name)).includes(groupsRefItem));
const groups = await DESEntitlement.getUserGroups(userToken, entitlementTenant, appkey);
return groupEmails.some((groupEmail) => (groups.map((group) => group.email)).includes(groupEmail));
}
public static async isMemberOfaGroup(
......
src/cloud/config.ts
View file @ f0b931c9
......@@ -85,9 +85,6 @@ export abstract class Config implements IConfig {
// Impersonation Token Service Account [this is the account used to sign the impersonation token]
public static IMP_SERVICE_ACCOUNT_SIGNER: string;
// Consistency Data Object
public static FILE_CDO = 'dataconsistency.cdo';
// Redis cache for lock
public static LOCKSMAP_REDIS_INSTANCE_ADDRESS: string;
public static LOCKSMAP_REDIS_INSTANCE_PORT: number;
......@@ -128,6 +125,14 @@ export abstract class Config implements IConfig {
public static FEATURE_FLAG_LOGGING = true;
public static FEATURE_FLAG_STACKDRIVER_EXPORTER = true;
// WriteLock Skip
// This is an open issue to discuss.
// Checking the write lock is the correct behaviour and this varialbe shoudl be set to "false".
// The current client libraries are not capable to send the lockin session id on mutable operations.
// As results imposing this check will break the functionalities of many current running applications.
// The C++ SDK mainly reuqire a fix on how behave on mutable calls.
public static SKIP_WRITE_LOCK_CHECK_ON_MUTABLE_OPERATIONS = true;
public static setCloudProvider(cloudProvider: string) {
Config.CLOUDPROVIDER = cloudProvider;
if (Config.CLOUDPROVIDER === undefined) {
......
src/cloud/credentials.ts
View file @ f0b931c9
......@@ -23,10 +23,7 @@ export interface IAccessTokenModel {
}
export interface ICredentials {
// [REVERT-DOWNSCOPE] add the following line getStorageCredentials
// getStorageCredentials(bucket: string, readonly: boolean): Promise<IAccessTokenModel>;
// [REVERT-DOWNSCOPE] remove the next line getUserCredentials
getUserCredentials(subject: string): Promise<IAccessTokenModel>;
getStorageCredentials(bucket: string, readonly: boolean, partitionID: string): Promise<IAccessTokenModel>;
getServiceAccountAccessToken(): Promise<IAccessTokenModel>;
getIAMResourceUrl(serviceSigner: string): string;
getAudienceForImpCredentials(): string;
......@@ -34,11 +31,9 @@ export interface ICredentials {
}
export abstract class AbstractCredentials implements ICredentials {
// [REVERT-DOWNSCOPE] add the following line getStorageCredentials
// public abstract async getStorageCredentials(bucket: string, readonly: boolean): Promise<IAccessTokenModel>;
// [REVERT-DOWNSCOPE] remove the next line getUserCredentials
public abstract async getUserCredentials(subject: string): Promise<IAccessTokenModel>;
public abstract async getServiceAccountAccessToken(): Promise<IAccessTokenModel>;
public abstract getStorageCredentials(
bucket: string, readonly: boolean, partitionID: string): Promise<IAccessTokenModel>;
public abstract getServiceAccountAccessToken(): Promise<IAccessTokenModel>;
public abstract getIAMResourceUrl(serviceSigner: string): string;
public abstract getAudienceForImpCredentials(): string;
public abstract getPublicKeyCertificatesUrl(): string;
......
src/cloud/providers/azure/cloudstorage.ts
View file @ f0b931c9
......@@ -17,12 +17,12 @@
import { TokenCredential } from '@azure/identity';
import { BlobServiceClient } from '@azure/storage-blob';
import { Readable } from 'stream';
import { TenantModel } from '../../../services/tenant';
import { Config } from '../../config';
import { AzureCredentials } from './credentials';
import { AbstractStorage, StorageFactory } from '../../storage';
import { AzureCredentials } from './credentials';
import { AzureDataEcosystemServices } from './dataecosystem';
import { TenantModel } from '../../../services/tenant';
@StorageFactory.register('azure')
export class AzureCloudStorage extends AbstractStorage {
......@@ -58,13 +58,8 @@ export class AzureCloudStorage extends AbstractStorage {
}
// Create a new container
// [REVERT-DOWNSCOPE] change method signature with the commented one
// public async createBucket(
// bucketName: string, location: string, storageClass: string): Promise<void> {
public async createBucket(
bucketName: string,
location: string, storageClass: string,
adminACL: string, editorACL: string, viewerACL: string): Promise<void> {
bucketName: string, location: string, storageClass: string): Promise<void> {
const container = (await this.getBlobServiceClient()).getContainerClient(bucketName);
await container.create();
}
......
src/cloud/providers/azure/credentials.ts
View file @ f0b931c9
......@@ -108,36 +108,10 @@ export class AzureCredentials extends AbstractCredentials {
}
}
// [REVERT-DOWNSCOPE] add this method
// public async getStorageCredentials(
// bucket: string,
// readonly: boolean)
// : Promise<IAccessTokenModel> {
// const accountName = AzureConfig.STORAGE_ACCOUNT_NAME;
// const now = new Date();
// const expiration = this.addMinutes(now, SasExpirationInMinutes);
// const sasToken = await this.generateSASToken(accountName, bucket, expiration, readonly);
// const result = {
// access_token: sasToken,
// expires_in: 3599,
// token_type: 'SasUrl',
// };
// return result;
// }
// [REVERT-DOWNSCOPE] remove this method
public async getUserCredentials(
subject: string)
: Promise<IAccessTokenModel> {
const accountName = await AzureDataEcosystemServices.getStorageAccountName(
subject.substr(0, subject.indexOf(';')));
subject = subject.substr(subject.indexOf(';') + 1);
public async getStorageCredentials(bucket: string,readonly: boolean,partition: string): Promise<IAccessTokenModel> {
const accountName = await AzureDataEcosystemServices.getStorageAccountName(partition);
const now = new Date();
const expiration = this.addMinutes(now, SasExpirationInMinutes);
const readonly = subject[subject.length - 1] === '1'
const bucket = subject.slice(0, -1);
const sasToken = await this.generateSASToken(accountName, bucket, expiration, readonly);
const result = {
access_token: sasToken,
......
src/cloud/providers/azure/index.ts
View file @ f0b931c9
......@@ -21,3 +21,4 @@ export { AzureConfig } from './config';
export { AzureCredentials } from './credentials';
export { AzureTrace } from './trace';
export { AzureDataEcosystemServices } from './dataecosystem';
export { AzureSeistore } from './seistore';
\ No newline at end of file
src/cloud/providers/azure/seistore.ts 0 → 100644
View file @ f0b931c9
// ============================================================================
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// ============================================================================
import { SubProjectModel } from '../../../services/subproject';
import { AbstractSeistore, SeistoreFactory } from '../../seistore';
@SeistoreFactory.register('azure')
export class AzureSeistore extends AbstractSeistore {
public checkExtraSubprojectCreateParams(requestBody: any, subproject: SubProjectModel) { return; }
}
src/cloud/providers/google/config.ts
View file @ f0b931c9
......@@ -22,8 +22,6 @@ export class ConfigGoogle extends Config {
// scopes
public static GOOGLE_SCOPE_PLATFORM = 'https://www.googleapis.com/auth/cloud-platform';
// [REVERT-DOWNSCOPE] remove this scope
public static GOOGLE_SCOPE_FULLCONTROL = 'https://www.googleapis.com/auth/devstorage.full_control';
// endpoints
public static GOOGLE_EP_IAM = 'https://iam.googleapis.com/v1';
......@@ -51,7 +49,7 @@ export class ConfigGoogle extends Config {
public static API_BASE_URL_PATH = '/api/' + ConfigGoogle.API_VERSION;
// max len for a group name in DE
public static DES_GROUP_CHAR_LIMIT = 64;
public static DES_GROUP_CHAR_LIMIT = 128;
public async init(): Promise<void> {
......
src/cloud/providers/google/credentials.ts
View file @ f0b931c9
......@@ -37,70 +37,42 @@ const KExpiresMargin = 300; // 5 minutes
@CredentialsFactory.register('google')
export class Credentials extends AbstractCredentials {
// [REVERT-DOWNSCOPE] add method
// public async getStorageCredentials(bucket: string, readonly: boolean): Promise<IAccessTokenModel> {
// return {
// access_token: (
// await this.exchangeJwtWithDownScopedAccessToken(
// (await this.getServiceAccountAccessToken()).access_token, bucket, readonly)).access_token,
// expires_in: 3599,
// token_type: 'Bearer',
// };
// }
// [REVERT-DOWNSCOPE] add method
// private async exchangeJwtWithDownScopedAccessToken(accessToken: string,
// bucket: string, readonly: boolean): Promise<IDownScopedToken> {
// try {
// return JSON.parse(await request.post({
// form: {
// access_boundary: JSON.stringify({
// accessBoundaryRules : [{
// availablePermissions: [
// 'inRole:roles/' + (readonly ? 'storage.objectViewer' : 'storage.objectAdmin') ],
// availableResource : '//storage.googleapis.com/projects/_/buckets/' + bucket,
// }],
// }),
// grant_type: 'urn:ietf:params:oauth:grant-type:token-exchange',
// requested_token_type: 'urn:ietf:params:oauth:token-type:access_token',
// subject_token: accessToken,
// subject_token_type: 'urn:ietf:params:oauth:token-type:access_token',
// },
// headers: {
// 'Content-Type': 'application/x-www-form-urlencoded',
// },
// url: 'https://securetoken.googleapis.com/v2beta1/token',
// }));
// } catch (error) {
// throw (Error.makeForHTTPRequest(error));
// }
// }
// [REVERT-DOWNSCOPE] remove method
public async getUserCredentials(subject: string): Promise<IAccessTokenModel> {
const now = Math.floor(Date.now() / 1000);
this.serviceAccountEmail = await this.getServiceAccountEmail();
const svcToken = (await this.getServiceAccountAccessToken()).access_token;
const options = {
body: {
payload: JSON.stringify({
aud: ConfigGoogle.GOOGLE_EP_OAUTH2 + '/token',
exp: now + 3600,
iat: now,
iss: this.serviceAccountEmail,
scope: ConfigGoogle.GOOGLE_SCOPE_FULLCONTROL,
sub: subject,
}),
},
headers: {
'Authorization': 'Bearer ' + svcToken,
'Content-Type': 'application/json',
},
json: true,
url: ConfigGoogle.GOOGLE_EP_IAM + '/projects/' +
ConfigGoogle.SERVICE_CLOUD_PROJECT + '/serviceAccounts/' + this.serviceAccountEmail + ':signJwt',
public async getStorageCredentials(
bucket: string, readonly: boolean, _partition: string): Promise<IAccessTokenModel> {
return {
access_token: (
await this.exchangeJwtWithDownScopedAccessToken(
(await this.getServiceAccountAccessToken()).access_token, bucket, readonly)).access_token,
expires_in: 3599,
token_type: 'Bearer',
};
return await this.signJWT((await request.post(options)).signedJwt) as IAccessTokenModel;
}
private async exchangeJwtWithDownScopedAccessToken(accessToken: string,
bucket: string, readonly: boolean): Promise<IDownScopedToken> {
try {
return JSON.parse(await request.post({
form: {
access_boundary: JSON.stringify({
accessBoundaryRules : [{
availablePermissions: [
'inRole:roles/' + (readonly ? 'storage.objectViewer' : 'storage.objectAdmin') ],
availableResource : '//storage.googleapis.com/projects/_/buckets/' + bucket,
}],
}),
grant_type: 'urn:ietf:params:oauth:grant-type:token-exchange',
requested_token_type: 'urn:ietf:params:oauth:token-type:access_token',
subject_token: accessToken,
subject_token_type: 'urn:ietf:params:oauth:token-type:access_token',
},
headers: {
'Content-Type': 'application/x-www-form-urlencoded',
},
url: 'https://securetoken.googleapis.com/v2beta1/token',
}));
} catch (error) {
throw (Error.makeForHTTPRequest(error));
}
}
public async getServiceCredentials(): Promise<string> {
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment