.gitignore

@@ -13,7 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ============================================================================
-
+    
+    
 # IDE configurations
 .vscode
 dist/

README.md

@@ -7,3 +7,9 @@ Saving a dataset on a cloud-based storage, as single entity, may be a problem wh
 Seismic Store is a cloud-based solution composed by restful micro-services, client APIs and tools designed to implement a multi-object storage approach. The system saves objects that compose a dataset as a hierarchical data structure in a cloud storage and the dataset properties as a metadata entry in a no-relational catalogue. Having the datasets stored as multiple independent objects improve the overall performance, as generic I/O operations, for example read or write objects, can be easily parallelized.
 
 Seismic Store manages data authorization at service level by protecting access to storage bucket resources. Only service authorized users are enabled to directly access a storage resource. The service implements a mechanism that generates an “impersonation token” by authorizing long running/background production jobs to access data without requiring further user interactions.
+
+Seismic DMS is a software suite solution compose by multiple micro services:
+
+1. [seismic store service V3](app/sdms/README.md): a DMS designed to store and manage datasets on the cloud.
+2. [seismic store service V4](app/sdms-v4/README.md): a DMS designed to store and manage seismic domain data on the cloud.
+3. [filemetadata](app/filemetadata/README.md): a microservice designed to compute, retrieve and manage seismic header data.
\ No newline at end of file

app/sdms/.gitignore

@@ -31,8 +31,6 @@ newman
 
 .DS_Store
 .idea
-
+ 
 # backup files
-*.bak
-
-
+*.bak
\ No newline at end of file

app/sdms/devops/gcp/configmap/README.md

@@ -30,7 +30,7 @@ First you need to set variables in **values.yaml** file using any code editor. S
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|---------|
-**logLevel** | logging level | string | "INFO" | yes
+**logLevel** | logging level | string | "ERROR" | yes
 **cloudProvider** | cloud provider | string | "google" | yes
 **port** | port | string | "5000" | yes
 **desServiceHost** | base url of host | string | "https://des" | yes

app/sdms/devops/gcp/configmap/values.yaml

 data:
   # common configuration
-  logLevel: "INFO"
+  logLevel: "ERROR"
   cloudProvider: "google"
   port: "5000"
   desServiceHost: "https://des"

app/sdms/devops/osdu/cloud-providers/azure.yml

@@ -36,7 +36,23 @@ sdms_azure_containerize:
     RELEASE_SIDECAR_IMAGE: release-${CI_COMMIT_TAG}:${SDMS_SIDECAR_PROJECT_NAME}-${CI_COMMIT_TAG}
   script:
     - cd app/$SDMS_SERVICE
+    - sed -i "/anthos/d" src/cloud/providers/index.ts
+    - rm -Rf src/cloud/providers/anthos
+    - sed -i "/google/d" src/cloud/providers/index.ts
+    - rm -Rf src/cloud/providers/aws
+    - sed -i "/aws/d" src/cloud/providers/index.ts
+    - rm -Rf src/cloud/providers/google
+    - sed -i "/ibm/d" src/cloud/providers/index.ts
+    - rm -Rf src/cloud/providers/ibm
     - sed -i 's|#{SDMS_PREFIX}#|'$AZURE_SDMS_PREFIX'|' ./docs/api/openapi.osdu.yaml
+    - jq "del(.prisma)" package.json > cleaned_package.json
+    - mv cleaned_package.json package.json
+    - export directoryToScan=$PWD
+    - export currentDirectory=$PWD
+    - chmod +x devops/scripts/detect_unused_packages.sh
+    - devops/scripts/detect_unused_packages.sh
+    - chmod +x devops/scripts/clean_package_json.sh
+    - devops/scripts/clean_package_json.sh
     # Runtime image
     - docker build -t $SDMS_UTEST_RUNTIME_IMAGE --file devops/azure/runtime.Dockerfile .
     # Sidecar image
@@ -211,7 +227,7 @@ sdms_azure_deploy:
       - tags
 
 sdms_azure_test:
-  image: node
+  image: node:18
   extends: .azure_test
   needs: ["sdms_azure_deploy"]
   variables:
@@ -220,10 +236,30 @@ sdms_azure_test:
     AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET
     INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID
   script:
+    - apt-get update
+    - apt-get install -y jq
     - cd app/$SDMS_SERVICE
+    - sed -i "/anthos/d" src/cloud/providers/index.ts
+    - rm -Rf src/cloud/providers/anthos
+    - sed -i "/google/d" src/cloud/providers/index.ts
+    - rm -Rf src/cloud/providers/aws
+    - sed -i "/aws/d" src/cloud/providers/index.ts
+    - rm -Rf src/cloud/providers/google
+    - sed -i "/ibm/d" src/cloud/providers/index.ts
+    - rm -Rf src/cloud/providers/ibm
+    - sed -i 's|#{SDMS_PREFIX}#|'$AZURE_SDMS_PREFIX'|' ./docs/api/openapi.osdu.yaml
+    - jq "del(.prisma)" package.json > cleaned_package.json
+    - mv cleaned_package.json package.json
+    - export directoryToScan=$PWD
+    - export currentDirectory=$PWD
+    - chmod +x devops/scripts/detect_unused_packages.sh
+    - devops/scripts/detect_unused_packages.sh
+    - chmod +x devops/scripts/clean_package_json.sh
+    - devops/scripts/clean_package_json.sh
     - echo $svctoken > /dev/null
     - npm install -g newman
     - npm install -g husky
+    - npm ci
     - chmod +x ./tests/e2e/run_e2e_tests.sh
     - ./tests/e2e/run_e2e_tests.sh --seistore-svc-url="https://${AZURE_DNS_NAME}${AZURE_SDMS_PREFIX}" --seistore-svc-api-key="NA" --user-idtoken="$svctoken" --tenant="${E2E_TENANT}" --admin-email="${E2E_ADMIN_EMAIL}" --datapartition="${E2E_DATA_PARTITION}" --legaltag01="${E2E_LEGALTAG01}" --legaltag02="${E2E_LEGALTAG02}" --VCS-Provider="${ISGITLAB}"  --subproject="${E2E_SUBPROJECT}"
   only:

app/sdms/devops/osdu/cloud-providers/gcp.yml

@@ -245,6 +245,7 @@ sdms-osdu-gcp-anthos-test-python:
   extends: .osdu-gcp-anthos-test-python
   image: node
   needs: ["sdms-osdu-gcp-anthos-deploy-deployment"]
+  allow_failure: true
   before_script:
     - cd app/$SDMS_SERVICE
   script:

app/sdms/devops/osdu/cloud-providers/ibm.yml

@@ -40,7 +40,7 @@ sdms_ibm-deploy-devpri:
         - app/sdms/**/*
     
 sdms_ibm-test-py:
-  image: node
+  image: node:18
   stage: integration
   extends:
     - .ibm-test-py

app/sdms/devops/osdu/containerize/osdu.yml

@@ -3,6 +3,7 @@ sdms_push_runtime_image:
   services:
     - docker:19.03.12-dind
   tags: ["osdu-medium"]
+  needs: ['sdms_compile-and-unit-test']
   stage: containerize
   variables:
     SHA_IMAGE: ${SDMS_PROJECT_NAME}:${CI_COMMIT_REF_SLUG}-${CI_COMMIT_SHA}
@@ -11,6 +12,10 @@ sdms_push_runtime_image:
     - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
   script:
     # Gitlab Container Registry
+    - export directoryToScan=$PWD
+    - export currentDirectory=$PWD
+    - chmod +x devops/scripts/detect_unused_packages.sh
+    - devops/scripts/detect_unused_packages.sh
     - cd app/$SDMS_SERVICE
     - docker build -t $SDMS_UTEST_RUNTIME_IMAGE --file docker/runtime.Dockerfile  .
     - docker tag $SDMS_UTEST_RUNTIME_IMAGE $CI_REGISTRY_IMAGE/$SHA_IMAGE
@@ -25,3 +30,4 @@ sdms_push_runtime_image:
       - branches
       - main
       - merge_requests
+      - tags
\ No newline at end of file

app/sdms/devops/scripts/clean_package_json.sh

+#!/bin/bash
+cd $directoryToScan
+
+lines=$(cat ${currentDirectory}/unused.txt)
+for line in $lines
+do
+  jq "del(.dependencies.\"${line}\")" package.json > cleaned_package.json
+  mv cleaned_package.json package.json
+  jq "del(.devDependencies.\"${line}\")" package.json > cleaned_package.json
+  mv cleaned_package.json package.json
+done
+
+cat package.json
+
+cd $currentDirectory
\ No newline at end of file

app/sdms/devops/scripts/detect_unused_packages.sh

+#!/bin/bash
+cd $directoryToScan
+
+FILES=$(mktemp)
+PACKAGES=$(mktemp)
+
+function check {
+  cat package.json | jq "{} + .$1 | keys" | sed -n 's/.*"\(.*\)".*/\1/p' > $PACKAGES
+  find . -type f -name '*.ts' > $FILES
+  find . -type f -name 'package.json' >> $FILES
+  while read PACKAGE
+  do
+    if ! grep -q $PACKAGE "devops/scripts/exclusions_detect_unused_packages.txt"
+	then
+      if [ -d "node_modules/${PACKAGE}" ]
+	  then
+        find node_modules/${PACKAGE} -type f -name '*.ts' >> $FILES
+	    find node_modules/${PACKAGE} -type f -name 'package.json' >> $FILES
+      fi
+      RES=$(cat $FILES | xargs -I {} egrep -i "(import|require|loader|plugins|${PACKAGE}).*['\"](${PACKAGE}|.?\d+)[\"']" '{}' | wc -l)
+      if [ $RES = 0 ]
+      then
+        echo -e "$PACKAGE"
+      fi
+	fi
+  done < $PACKAGES
+}
+
+check "dependencies" > $currentDirectory/unused.txt
+check "devDependencies" >> $currentDirectory/unused.txt
+check "peerDependencies" >> $currentDirectory/unused.txt
+
+cd $currentDirectory
+
+cat unused.txt
\ No newline at end of file

app/sdms/devops/scripts/exclusions_detect_unused_packages.txt

+DEPENDENCIES
+
+applicationinsights-native-metrics
+colors
+copyfiles
+dotenv
+typescript
+
+DEV DEPENDENCIES
+
+@types/bull
+@types/chai
+@types/cors
+@types/express
+@types/ioredis
+@types/js-yaml
+@types/jsonwebtoken
+@types/mocha
+@types/node
+@types/redlock
+@types/sinon
+@types/swagger-ui-express
+@types/uuid
+@types/xss-filters
+istanbul
+mocha-bamboo-reporter
+mocha-junit-reporter
+newman
+nyc
+ts-node
\ No newline at end of file

app/sdms/docs/api/openapi.osdu.yaml

@@ -1420,30 +1420,30 @@ paths:
 
   /subproject/tenant/{tenantid}/subproject/{subprojectid}:
     post:
-      summary: "Create a new subproject."
+      summary: 'Create a new subproject.'
       description: |
         <ul>
-        <li>Creates a new sub-project resource in seismic store.</li>
+        <li>Creates a new subproject resource in seismic store.</li>
         <li>Required roles: users.datalake.admin</li></ul>
       operationId: subproject-create
       tags:
         - SubProject
       parameters:
-        - description: "Legal tag of the datapartition/tenant."
+        - description: 'A valid compliance legal tag. It can be generated and managed via core-compliance-service.'
           in: header
           name: ltag
           type: string
-        - description: "Name of the subproject."
+        - description: 'The subproject name. It must match the regex <b>^[a-z][a-z\d\-]*[a-z\d]$</b>'
           in: path
           name: subprojectid
           type: string
           required: true
-        - description: "Name of the tenant."
+        - description: 'The tenant name - In OSDU this match the data-partition-id'
           in: path
           name: tenantid
           type: string
           required: true
-        - description: "Request body"
+        - description: 'Request body'
           in: body
           name: request body
           required: true
@@ -1451,7 +1451,7 @@ paths:
             $ref: "#/definitions/SubProjectCreateBody"
       responses:
         200:
-          description: "Creation of the subproject is successful."
+          description: 'The registered subproject.'
           schema:
             $ref: "#/definitions/SubProject"
         400:
@@ -1502,7 +1502,6 @@ paths:
           description: "Forbidden."
         404:
           description: "Not found."
-
     delete:
       summary: "Delete a subproject."
       description: "<ul><li>Delete a subproject in seismic store.</li><li>Required roles: users.datalake.admin</li></ul>"
@@ -1531,7 +1530,6 @@ paths:
           description: "Forbidden."
         404:
           description: "Not found."
-
     patch:
       summary: "Patch a subproject's metadata."
       description: |
@@ -1710,7 +1708,7 @@ paths:
   /user:
     put:
       summary: "Add a user to a seismic store subproject authorization group."
-      description: "<ul><li>Add a user to a subproject authorization group.</li><li>Required roles: subproject.admin</li></ul>"
+      description: "<ul><li>Add a user to a subproject default authorization group if it exists, otherwise, add the user to the first group.</li><li>Required roles: subproject.admin</li></ul>"
       operationId: user-add
       tags:
         - Users
@@ -1766,8 +1764,8 @@ paths:
           description: "Not found."
 
     delete:
-      summary: "Remove a user from a subproject's role-based authorization groups."
-      description: "<ul><li>Remove a user from a subproject's role based authorization groups.</li><li>Required roles: subproject.admin</li></ul>"
+      summary: "Remove a user from a subproject."
+      description: "<ul><li>Remove a user from subproject default authorization groups if exists, otherwise, remove it from the first authorization group.</li><li>Required roles: subproject.admin</li></ul>"
       operationId: user-rm
       tags:
         - Users
@@ -1779,7 +1777,7 @@ paths:
             $ref: "#/definitions/UserRemoveRequest"
       responses:
         200:
-          description: Deletion of user's email in the subproject's role-based authorization groups completed."
+          description: Deletion of user from a subproject."
         400:
           description: "Bad request."
         401:
@@ -2407,25 +2405,24 @@ definitions:
 
   # OK
   SubProjectCreateBody:
-    required: ["admin", "storage_class", "storage_location", "access_policy"]
     properties:
       admin:
         type: string
-        description: Administrator for the subproject.
+        description: 'An optional user to set as admin. The user will be added in the first admins acl group'
       storage_class:
         type: string
-        description: Storage class for the bucket.
+        description: 'Storage class for the bucket (Google Required Only)'
       storage_location:
         type: string
-        description: Storage location for the bucket.
+        description: 'Storage location for the bucket (Google Required Only)'
       access_policy:
         type: string
-        description: Access policy for the subproject.
+        description: 'The datasets access level mode: "uniform" (uniform data access to all subprojects datasets) or "datasets" (acl can be applied at dataset level)'
         default: "uniform"
-        enum: [uniform, dataset]
+        enum: ["uniform", "dataset"]
       acls:
         type: object
-        description: ACLs with admin groups and viewer groups for the subproject.
+        description: 'The entitlement groups to enable subproject access as admin or viewer. If not specified, default entitlement data groups will be created.'
         properties:
           admins:
             type: array
@@ -2436,9 +2433,10 @@ definitions:
             items:
               type: string
     example:
-      admin: "test@email"
+      admin: "admin@email"
       storage_class: "MULTI_REGIONAL"
       storage_location: "US"
+      access_policy: "uniform"
       acls:
         admins:
           - data.sdms.tenant.subproject.1b1417e5-fb0e-402a-b725-606723e50b76.admin@slb.p4d.cloud.slb-ds.com
@@ -2475,7 +2473,7 @@ definitions:
 
   # OK
   SubProject:
-    required: ["name", "tenant", "storage_class", "storage_location", "admin", "ltag", "gcs_bucket"]
+    required: ["name", "tenant", "storage_class", "storage_location", "ltag", "access_policy"]
     properties:
       name:
         type: string
@@ -2489,15 +2487,14 @@ definitions:
       storage_location:
         type: string
         description: Storage location for the bucket.
-      admin:
-        type: string
-        description: Administrator for the subproject. Note that this field is used as a principal identifer for entitlements-svc and be configured using the env variable USER_ID_CLAIM_FOR_ENTITLEMENTS_SVC in the Seismic DMS runtime.
       ltag:
         type: string
         description: Legal tag for the subproject.
-      gcs_bucket:
+      access_policy:
         type: string
-        description: Cloud storage bucket associated with the subproject.
+        description: Access policy for the subproject.
+        default: "uniform"
+        enum: [uniform, dataset]
       acls:
         type: object
         description: ACLs with admin groups and viewer groups for the subproject.

app/sdms/docs/api/openapi.yaml

@@ -1470,33 +1470,30 @@ paths:
 
   /api/v3/subproject/tenant/{tenantid}/subproject/{subprojectid}:
     post:
-      summary: "Create a new subproject."
+      summary: 'Create a new subproject.'
       description: |
         <ul>
-        <li>Creates a new sub-project resource in seismic store.</li>
+        <li>Creates a new subproject resource in seismic store.</li>
         <li>Required roles: users.datalake.admin</li></ul>
       operationId: subproject-create
       tags:
         - SubProject
       parameters:
-        - description: "Legal tag of the datapartition/tenant."
+        - description: 'A valid compliance legal tag. It can be generated and managed via core-compliance-service.'
           in: header
           name: ltag
           type: string
-          default: "Slb-Private-USA-EHC"
-        - description: "Name of the subproject."
+        - description: 'The subproject name. It must match the regex <b>^[a-z][a-z\d\-]*[a-z\d]$</b>'
           in: path
           name: subprojectid
           type: string
           required: true
-          default: carbon
-        - description: "Name of the tenant."
+        - description: 'The tenant name'
           in: path
           name: tenantid
           type: string
           required: true
-          default: common
-        - description: "Request body"
+        - description: 'Request body'
           in: body
           name: request body
           required: true
@@ -1504,7 +1501,7 @@ paths:
             $ref: "#/definitions/SubProjectCreateBody"
       responses:
         200:
-          description: "Creation of the subproject is successful."
+          description: 'The registered subproject.'
           schema:
             $ref: "#/definitions/SubProject"
         400:
@@ -1770,8 +1767,8 @@ paths:
 
   /api/v3/user:
     put:
-      summary: "Add a user to a seismic store subproject authorization group."
-      description: "<ul><li>Add a user to a subproject authorization group.</li><li>Required roles: subproject.admin</li></ul>"
+      summary: "Add a user to a seismic store subproject authorization group." 
+      description: "<ul><li>Add a user to a subproject default authorization group if it exists, otherwise, add the user to the first group.</li><li>Required roles: subproject.admin</li></ul>"
       operationId: user-add
       tags:
         - Users
@@ -1828,8 +1825,8 @@ paths:
           description: "Not found."
 
     delete:
-      summary: "Remove a user from a subproject's role-based authorization groups."
-      description: "<ul><li>Remove a user from a subproject's role based authorization groups.</li><li>Required roles: subproject.admin</li></ul>"
+      summary: "Remove a user from a subproject."\
+      description: "<ul><li>Remove a user from subproject default authorization groups if exists, otherwise, remove it from the first authorization group.</li><li>Required roles: subproject.admin</li></ul>"
       operationId: user-rm
       tags:
         - Users
@@ -1841,7 +1838,7 @@ paths:
             $ref: "#/definitions/UserRemoveRequest"
       responses:
         200:
-          description: Deletion of user's email in the subproject's role-based authorization groups completed."
+          description: Deletion of user from a subproject."
         400:
           description: "Bad request."
         401:
@@ -2444,25 +2441,24 @@ definitions:
 
   # OK
   SubProjectCreateBody:
-    required: ["admin", "storage_class", "storage_location", "access_policy"]
     properties:
       admin:
         type: string
-        description: Administrator for the subproject.
+        description: 'An optional user to set as admin. The user will be added in the first admins acl group'
       storage_class:
         type: string
-        description: Storage class for the bucket.
+        description: 'Storage class for the bucket (Google Required Only)'
       storage_location:
         type: string
-        description: Storage location for the bucket.
+        description: 'Storage location for the bucket (Google Required Only)'
       access_policy:
         type: string
-        description: Access policy for the subproject.
+        description: 'The datasets access level mode: "uniform" (uniform data access to all subprojects datasets) or "datasets" (acl can be applied at dataset level)'
         default: "uniform"
-        enum: [uniform, dataset]
+        enum: ["uniform", "dataset"]
       acls:
         type: object
-        description: ACLs with admin groups and viewer groups for the subproject.
+        description: 'The entitlement groups to enable subproject access as admin or viewer. If not specified, default entitlement data groups will be created.'
         properties:
           admins:
             type: array
@@ -2473,15 +2469,17 @@ definitions:
             items:
               type: string
     example:
-      admin: "test@email"
+      admin: "admin@email"
       storage_class: "MULTI_REGIONAL"
       storage_location: "US"
+      access_policy: "uniform"
       acls:
         admins:
           - data.sdms.tenant.subproject.1b1417e5-fb0e-402a-b725-606723e50b76.admin@slb.p4d.cloud.slb-ds.com
         viewers:
           - data.sdms.tenant.subproject.1b1417e5-fb0e-402a-b725-606723e50b76.viewer@slb.p4d.cloud.slb-ds.com
 
+
   # Subproject patch operation request body
   SubProjectPatchBody:
     properties:
@@ -2512,7 +2510,7 @@ definitions:
 
   # OK
   SubProject:
-    required: ["name", "tenant", "storage_class", "storage_location", "admin", "ltag", "gcs_bucket"]
+    required: ["name", "tenant", "storage_class", "storage_location", "ltag", "access_policy"]
     properties:
       name:
         type: string
@@ -2526,15 +2524,14 @@ definitions:
       storage_location:
         type: string
         description: Storage location for the bucket.
-      admin:
-        type: string
-        description: Administrator for the subproject. Note that this field is used as a principal identifer for entitlements-svc and be configured using the env variable USER_ID_CLAIM_FOR_ENTITLEMENTS_SVC in the Seismic DMS runtime.
       ltag:
         type: string
         description: Legal tag for the subproject.
-      gcs_bucket:
+      access_policy:
         type: string
-        description: Cloud storage bucket associated with the subproject.
+        description: Access policy for the subproject.
+        default: "uniform"
+        enum: [uniform, dataset]
       acls:
         type: object
         description: ACLs with admin groups and viewer groups for the subproject.

app/sdms/package.json

@@ -27,11 +27,10 @@
   },
   "nyc": {
     "check-coverage": true,
-    "branches": 24,
+    "branches": 23,
     "functions": 30,
-    "lines": 34,
-    "statements": 37,
-    
+    "lines": 33,
+    "statements": 35,
     "include": [
       "src/**/*.ts"
     ],
@@ -60,27 +59,21 @@
   "dependencies": {
     "@aws-sdk/client-s3": "^3.49.0",
     "@aws-sdk/client-sts": "^3.49.0",
-    "@aws-sdk/signature-v4": "^3.49.0",
+    "@azure/core-auth": "^1.4.0",
+    "@azure/core-rest-pipeline": "^1.9.2",
     "@azure/cosmos": "^3.14.1",
     "@azure/identity": "^2.0.1",
     "@azure/keyvault-secrets": "^4.3.0",
     "@azure/storage-blob": "^12.10.0",
     "@cloudant/cloudant": "^4.5.1",
     "@google-cloud/datastore": "^6.6.2",
-    "@google-cloud/logging": "^10.0.3",
     "@google-cloud/logging-winston": "^5.1.0",
     "@google-cloud/pubsub": "^3.0.1",
     "@google-cloud/secret-manager": "^4.0.0",
     "@google-cloud/storage": "^6.1.0",
     "@google-cloud/trace-agent": "^5.1.6",
-    "@mapbox/node-pre-gyp": "^1.0.6",
     "@prisma/client": "^3.8.0",
-    "@types/ioredis": "^4.28.1",
-    "@types/js-yaml": "^4.0.5",
-    "@types/redlock": "^4.0.2",
-    "@types/yamljs": "^0.2.31",
     "ajv": "^8.8.0",
-    "ajv-errors": "^3.0.0",
     "ajv-formats": "^2.1.1",
     "applicationinsights": "^2.1.8",
     "applicationinsights-native-metrics": "0.0.6",
@@ -92,7 +85,6 @@
     "cors": "^2.8.5",
     "dotenv": "^10.0.0",
     "express": "4.17.1",
-    "extend": "^3.0.2",
     "hpropagate": "^1.0.1",
     "ioredis": "^5.1.0",
     "js-yaml": "^4.1.0",
@@ -100,23 +92,17 @@
     "jsonwebtoken": "8.5.1",
     "jwtproxy": "^1.6.8",
     "keycloak-admin": "^1.14.22",
-    "lodash": "^4.17.21",
     "log4js": "^6.3.0",
-    "logform": "^2.3.0",
-    "minimist": "^1.2.6",
-    "mkdirp": "^1.0.4",
-    "prisma": "^3.8.0",
     "node-cache": "^5.1.2",
+    "prisma": "^3.8.0",
+    "qs": "^6.11.0",
     "redlock": "^5.0.0-beta.2",
     "replace-in-file": "^6.3.2",
     "swagger-ui-express": "^4.3.0",
     "typescript": "^4.4.4",
     "uuid": "^8.3.2",
     "winston": "3.3.3",
-    "xss-filters": "1.2.7",
-    "yamljs": "0.3.0",
-    "yargs": "^17.2.1",
-    "yargs-parser": "^20.2.9"
+    "xss-filters": "1.2.7"
   },
   "devDependencies": {
     "@commitlint/cli": "^14.1.0",
@@ -125,10 +111,12 @@
     "@types/chai": "^4.2.22",
     "@types/cors": "^2.8.12",
     "@types/express": "^4.17.13",
+    "@types/ioredis": "^4.28.1",
+    "@types/js-yaml": "^4.0.5",
     "@types/jsonwebtoken": "^8.5.5",
     "@types/mocha": "^9.0.0",
     "@types/node": "^16.11.7",
-    "@types/redis": "^2.8.32",
+    "@types/redlock": "^4.0.2",
     "@types/sinon": "^10.0.6",
     "@types/swagger-ui-express": "^4.1.3",
     "@types/uuid": "^8.3.1",
@@ -138,24 +126,19 @@
     "commitizen": "^4.2.4",
     "cz-conventional-changelog": "^3.3.0",
     "fs-jetpack": "^4.2.0",
-    "handlebars": "^4.7.7",
     "husky": "^7.0.4",
     "inline-css": "^3.0.0",
     "ioredis-mock": "^8.2.2",
     "istanbul": "0.4.5",
-    "lodash.merge": "^4.6.2",
     "mocha": "^9.1.3",
     "mocha-bamboo-reporter": "^1.1.2",
     "mocha-junit-reporter": "^2.0.2",
     "newman": "^5.3.0",
     "nyc": "^15.1.0",
-    "path": "^0.12.7",
     "scan-for-secrets": "^2.0.3",
     "sinon": "^12.0.1",
-    "source-map-support": "^0.5.20",
     "ts-node": "^10.4.0",
-    "tslint": "^6.1.3",
-    "tslint-no-unused-expression-chai": "0.1.4"
+    "tslint": "^6.1.3"
   },
   "config": {
     "commitizen": {
@@ -165,4 +148,4 @@
   "prisma": {
     "schema": "src/cloud/providers/anthos/schema.prisma"
   }
-}
\ No newline at end of file
+}

app/sdms/src/cloud/providers/azure/sidecar/Dockerfile

@@ -23,6 +23,7 @@ RUN dotnet dev-certs https --trust
 FROM build AS publish
 RUN dotnet publish "Sidecar.csproj" -c Release -o /app/publish /p:UseAppHost=false
 RUN cp ~/.aspnet/https/aspnetapp.pfx /app/publish/.
+RUN chmod 755 /app/publish/aspnetapp.pfx
 
 FROM base AS final
 ARG PASS

app/sdms/src/cloud/providers/google/credentials.ts

@@ -206,7 +206,7 @@ export class Credentials extends AbstractCredentials {
         const url = ConfigGoogle.GOOGLE_EP_METADATA + '/instance/service-accounts/default/email';
 
         try {
-            await axios.get(url, options);
+            return (await axios.get(url, options)).data;
         } catch (error) {
             throw (Error.makeForHTTPRequest(error));
         }

app/sdms/src/cloud/providers/ibm/config.ts

@@ -26,7 +26,9 @@ export class IbmConfig extends Config {
     public static COS_SIGNATUREVERSION: string;
     public static COS_SUBUSER_ACCESS_KEY_ID: string;
     public static COS_SUBUSER_SECRET_ACCESS_KEY: string;
-    public static COS_TEMP_CRED_EXPITY: string;
+    public static COS_TEMP_CRED_EXPIRY: string;
+    public static COS_ROLE_ARN: string;
+    public static COS_ROLE_SESSION_NAME: string;
 
     // IBM KeyCloak
     public static KEYCLOAK_BASEURL: string;
@@ -95,7 +97,9 @@ export class IbmConfig extends Config {
         IbmConfig.COS_SIGNATUREVERSION = process.env.COS_SIGNATUREVERSION;
         IbmConfig.COS_SUBUSER_ACCESS_KEY_ID = process.env.COS_SUBUSER_ACCESS_KEY_ID;
         IbmConfig.COS_SUBUSER_SECRET_ACCESS_KEY = process.env.COS_SUBUSER_SECRET_ACCESS_KEY;
-        IbmConfig.COS_TEMP_CRED_EXPITY = process.env.COS_TEMP_CRED_EXPITY;
+        IbmConfig.COS_TEMP_CRED_EXPIRY = process.env.COS_TEMP_CRED_EXPIRY || '7200';
+        IbmConfig.COS_ROLE_ARN = process.env.COS_ROLE_ARN || 'arn:aws:iam:::role/osdurolearn';
+        IbmConfig.COS_ROLE_SESSION_NAME = process.env.COS_ROLE_SESSION_NAME || 'Bob';
 
         // IBM Keycloak
         IbmConfig.KEYCLOAK_BASEURL = process.env.KEYCLOAK_BASEURL;

app/sdms/src/cloud/providers/ibm/credentials.ts

@@ -48,8 +48,9 @@ export class Credentials extends AbstractCredentials {
         tenant: string, subproject: string,
         bucket: string, readonly: boolean, _partition: string): Promise<IAccessTokenModel> {
 
-        const expDuration = IbmConfig.COS_TEMP_CRED_EXPITY;
+        const expDuration = IbmConfig.COS_TEMP_CRED_EXPIRY;
         let roleArn = '';
+        let roleSessionName = '';
         let credentials = '';
 
         let flagUpload = true;
@@ -58,19 +59,21 @@ export class Credentials extends AbstractCredentials {
         // this can start getting folder from gcs url along with bucket
         const s3bucket = keyPath;
 
+        roleArn = IbmConfig.COS_ROLE_ARN;
+        roleSessionName = IbmConfig.COS_ROLE_SESSION_NAME;
+
         if (readonly) { // readOnly True
-            roleArn = 'arn:123:456:789:1234';
             flagUpload = false;
         } else {// readOnly False
-            roleArn = 'arn:123:456:789:1234';
             flagUpload = true;
         }
 
-        credentials = await this.ibmSTSHelper.getCredentials(s3bucket, keyPath, roleArn, flagUpload, expDuration);
+        credentials = await this.ibmSTSHelper.getCredentials(s3bucket, keyPath,
+            roleArn, roleSessionName, flagUpload, expDuration);
 
         const result = {
             access_token: credentials,
-            expires_in: 7200,
+            expires_in: parseInt(expDuration, 10),
             token_type: 'Bearer',
         };