Commit 45b61ab9 authored by Diego Molteni's avatar Diego Molteni
Browse files

Merge branch 'master' into slb/dm3/fix-old-imptoken

parents 87ce5e04 6c4fca24
Pipeline #59208 passed with stages
in 12 minutes and 38 seconds
......@@ -5,6 +5,8 @@ variables:
PORT: 80
REPLICA: 1
UTEST_RUNTIME_IMAGE: seistore-svc-runtime
SDMS_MIN_REPLICAS: 1
SDMS_MAX_REPLICAS: 5
#aws variables
AWS_SERVICE: seismic-store
......@@ -34,7 +36,7 @@ variables:
OSDU_GCP_APPLICATION: seismic-store
OSDU_GCP_ENTITLEMENT_BASE_URL_PATH: /entitlements/v2
OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY: data-partition-id
OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE: https://os-legal-attcrcktoa-uc.a.run.app/api
OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE: https://community.osdu-gcp.go3-nrg.projects.epam.com/api
OSDU_GCP_DES_SERVICE_HOST_STORAGE: https://os-storage-attcrcktoa-uc.a.run.app/api
OSDU_GCP_ENV_VARS: CLOUDPROVIDER=${OSDU_GCP_CLOUD_PROVIDER},DES_SERVICE_HOST_PARTITION=${OSDU_GCP_PARTITION_API},ENTITLEMENT_BASE_URL_PATH=${OSDU_GCP_ENTITLEMENT_BASE_URL_PATH},DATA_PARTITION_REST_HEADER_KEY=${OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY},DES_SERVICE_HOST_STORAGE=${OSDU_GCP_DES_SERVICE_HOST_STORAGE},DES_SERVICE_HOST_COMPLIANCE=${OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE},SEISTORE_DES_TARGET_AUDIENCE=${GOOGLE_AUDIENCE},SERVICE_CLOUD_PROJECT=${OSDU_GCP_PROJECT},APP_ENVIRONMENT_IDENTIFIER=${TENANT},IMP_SERVICE_ACCOUNT_SIGNER=${OSDU_GCP_IMP_SERVICE_ACCOUNT_SIGNER},DES_SERVICE_HOST_ENTITLEMENT=${OSDU_GCP_ENTITLEMENTS_V2_BASE_URL},SEISTORE_DES_APPKEY=${OSDU_GCP_SEISTORE_DES_APPKEY},DES_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_DES_REDIS_INSTANCE_ADDRESS},DES_REDIS_INSTANCE_PORT=${OSDU_GCP_DES_REDIS_INSTANCE_PORT},LOCKSMAP_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_LOCKSMAP_REDIS_INSTANCE_ADDRESS} --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
......@@ -52,6 +54,9 @@ include:
# lint
- local: "/devops/osdu/scanners/lint-node.yml"
# scan for secrets
- local: "/devops/osdu/scanners/scan-for-secrets-node.yml"
# containerize
- project: "osdu/platform/ci-cd-pipelines"
......
......@@ -468,7 +468,7 @@ The following software have components provided under the terms of this license:
- cross-spawn (from https://www.npmjs.com/package/cross-spawn)
- crypto-random-string (from https://www.npmjs.com/package/crypto-random-string)
- d64 (from https://www.npmjs.com/package/d64)
- dashdash (from https://github.com/trentm/node-dashdash)
- dashdash (from https://www.npmjs.com/package/dashdash)
- date-and-time (from https://www.npmjs.com/package/date-and-time)
- date-format (from https://www.npmjs.com/package/date-format)
- date-format (from https://www.npmjs.com/package/date-format)
......@@ -511,7 +511,7 @@ The following software have components provided under the terms of this license:
- execa (from https://www.npmjs.com/package/execa)
- express (from https://www.npmjs.com/package/express)
- extend (from https://www.npmjs.com/package/extend)
- extsprintf (from https://github.com/davepacheco/node-extsprintf)
- extsprintf (from https://www.npmjs.com/package/extsprintf)
- fast-deep-equal (from https://www.npmjs.com/package/fast-deep-equal)
- fast-json-stable-stringify (from https://www.npmjs.com/package/fast-json-stable-stringify)
- fast-safe-stringify (from https://www.npmjs.com/package/fast-safe-stringify)
......@@ -708,7 +708,7 @@ The following software have components provided under the terms of this license:
- performance-now (from https://www.npmjs.com/package/performance-now)
- priorityqueuejs (from https://www.npmjs.com/package/priorityqueuejs)
- process (from https://www.npmjs.com/package/process)
- process-nextick-args (from https://github.com/calvinmetcalf/process-nextick-args)
- process-nextick-args (from https://www.npmjs.com/package/process-nextick-args)
- promise.prototype.finally (from https://www.npmjs.com/package/promise.prototype.finally)
- protobufjs (from https://www.npmjs.com/package/protobufjs)
- proxy-addr (from https://www.npmjs.com/package/proxy-addr)
......@@ -815,7 +815,7 @@ The following software have components provided under the terms of this license:
- universalify (from https://www.npmjs.com/package/universalify)
- unpipe (from https://github.com/stream-utils/unpipe)
- untildify (from https://www.npmjs.com/package/untildify)
- url (from https://github.com/defunctzombie/node-url#readme)
- url (from https://www.npmjs.com/package/url)
- url-join (from https://www.npmjs.com/package/url-join)
- urllib3 (from https://urllib3.readthedocs.io/)
- util-deprecate (from https://github.com/TooTallNate/util-deprecate)
......@@ -827,7 +827,7 @@ The following software have components provided under the terms of this license:
- uuid (from https://www.npmjs.com/package/uuid)
- uuid (from https://www.npmjs.com/package/uuid)
- vary (from https://www.npmjs.com/package/vary)
- verror (from https://github.com/davepacheco/node-verror)
- verror (from https://www.npmjs.com/package/verror)
- which (from https://github.com/isaacs/node-which#readme)
- which-module (from https://www.npmjs.com/package/which-module)
- wide-align (from https://www.npmjs.com/package/wide-align)
......@@ -930,3 +930,4 @@ The following software have components provided under the terms of this license:
- jmespath (from https://www.npmjs.com/package/jmespath)
- json-schema (from http://github.com/kriszyp/json-schema)
- querystring (from https://www.npmjs.com/package/querystring)
- sax (from https://www.npmjs.com/package/sax)
\ No newline at end of file
......@@ -2,6 +2,8 @@ global:
replicaCount: #{REPLICA_COUNT}#
namespace: osdu
podidentity: osdu-identity
minReplicaCount: #{SDMS_MIN_REPLICAS}#
maxReplicaCount: #{SDMS_MAX_REPLICAS}#
configEnv:
cloudProvider: #{PROVIDER_NAME}#
......
apiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
metadata:
name: {{ .Release.Name }}
namespace: osdu
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ .Release.Name }}
minReplicas: {{ .Values.global.minReplicaCount }}
maxReplicas: {{ .Values.global.maxReplicaCount }}
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 50
behavior:
scaleDown:
stabilizationWindowSeconds: 300
# Either remove 20% of current pods or 2 pods (whichever is lower) every 15 seconds until the the HPA stabilizes
selectPolicy: Min
policies:
- type: Percent
value: 20
periodSeconds: 15
- type: Pods
value: 2
periodSeconds: 15
scaleUp:
stabilizationWindowSeconds: 0
# Either add 100% of current pods or 4 pods (whichever is higher) every 1 second until the the HPA stabilizes
selectPolicy: Max
policies:
- type: Percent
value: 85
periodSeconds: 1
- type: Pods
value: 4
periodSeconds: 1
\ No newline at end of file
......@@ -12,5 +12,6 @@ spec:
- protocol: TCP
port: 80
targetPort: 80
name: http
selector:
app: {{ .Release.Name }}
\ No newline at end of file
app: {{ .Release.Name }}
......@@ -117,4 +117,12 @@ $ docker run --rm -it -v $(pwd):/opt community.opengroup.org:5555/osdu/platform/
```bash
$ docker run --rm -it -v $(pwd):/opt community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service/seismic-store-service-detect-secrets:latest detect-secrets-hook --baseline /opt/devops/docker/detect_secrets/.secrets.baseline $(git ls-files)
```
##### False positives
Add next comment above the line (in the proper file) that has been detected and is a false positives
```
pragma: allowlist nextline secret
```
\ No newline at end of file
......@@ -39,6 +39,8 @@ azure_deploy:
- sed -i 's/#{REDIS_HOST}#/'$REDIS_INSTANCE_ADDRESS'/' ${CHART_PATH}/values.yaml
- sed -i 's/#{REDIS_PORT}#/'$REDIS_INSTANCE_PORT'/' ${CHART_PATH}/values.yaml
- sed -i 's/#{REPLICA_COUNT}#/'$REPLICA'/' ${CHART_PATH}/values.yaml
- sed -i 's/#{SDMS_MIN_REPLICAS}#/'$SDMS_MIN_REPLICAS'/' ${CHART_PATH}/values.yaml
- sed -i 's/#{SDMS_MAX_REPLICAS}#/'$SDMS_MAX_REPLICAS'/' ${CHART_PATH}/values.yaml
# Install helm chart
- helm upgrade $SERVICE_NAME ${CHART_PATH} --install --dry-run --values $CHART_PATH/values.yaml
- helm upgrade $SERVICE_NAME ${CHART_PATH} --install --values $CHART_PATH/values.yaml
......
......@@ -2,6 +2,8 @@ scan-for-secrets:
image: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service/seismic-store-service-detect-secrets:latest
tags: ["osdu-small"]
stage: scan
needs: ['compile-and-unit-test']
needs:
- job: compile-and-unit-test
artifacts: false
script:
- detect-secrets-hook --exclude-files devops/docker/detect_secrets/.secrets.baseline --exclude-files devops/osdu/scanners/scan-for-secrets-node.yml --exclude-files npm-shrinkwrap.json --exclude-files package.json --exclude-files devops/scripts/azure_jwt_client.py --exclude-files src/cloud/providers/azure/keyvault.ts --exclude-files tests/utest/cloud/azure/keyvault.ts --baseline devops/docker/detect_secrets/.secrets.baseline $(git ls-files)
\ No newline at end of file
- detect-secrets-hook --exclude-files npm-shrinkwrap.json --exclude-files package.json --exclude-files devops/osdu/scanners/scan-for-secrets-node.yml --baseline devops/docker/detect_secrets/.secrets.baseline $(git ls-files)
\ No newline at end of file
......@@ -50,7 +50,8 @@ def get_invalid_token():
}
'''
return "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJkdW1teUBkdW1teS5jb20iLCJpc3MiOiJkdW1teUBkdW1teS5jb20iLCJhdWQiOiJkdW1teS5kdW1teS5jb20iLCJpYXQiOjE1NTYxMzcyNzMsImV4cCI6MTU1NjIzMDk3OSwicHJvdmlkZXIiOiJkdW1teS5jb20iLCJjbGllbnQiOiJkdW1teS5jb20iLCJ1c2VyaWQiOiJkdW1teXRlc3Rlci5jb20iLCJlbWFpbCI6ImR1bW15dGVzdGVyLmNvbSIsImF1dGh6IjoiIiwibGFzdG5hbWUiOiJkdW1teSIsImZpcnN0bmFtZSI6ImR1bW15IiwiY291bnRyeSI6IiIsImNvbXBhbnkiOiIiLCJqb2J0aXRsZSI6IiIsInN1YmlkIjoiZHVtbXlpZCIsImlkcCI6ImR1bW15IiwiaGQiOiJkdW1teS5jb20iLCJkZXNpZCI6ImR1bW15aWQiLCJjb250YWN0X2VtYWlsIjoiZHVtbXlAZHVtbXkuY29tIiwianRpIjoiNGEyMWYyYzItZjU5Yy00NWZhLTk0MTAtNDNkNDdhMTg4ODgwIn0.nkiyKtfXXxAlC60iDjXuB2EAGDfZiVglP-CyU1T4etc"
# pragma: allowlist nextline secret
return "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJkdW1teUBkdW1teS5jb20iLCJpc3MiOiJkdW1teUBkdW1teS5jb20iLCJhdWQiOiJkdW1teS5kdW1teS5jb20iLCJpYXQiOjE1NTYxMzcyNzMsImV4cCI6MTU1NjIzMDk3OSwicHJvdmlkZXIiOiJkdW1teS5jb20iLCJjbGllbnQiOiJkdW1teS5jb20iLCJ1c2VyaWQiOiJkdW1teXRlc3Rlci5jb20iLCJlbWFpbCI6ImR1bW15dGVzdGVyLmNvbSIsImF1dGh6IjoiIiwibGFzdG5hbWUiOiJkdW1teSIsImZpcnN0bmFtZSI6ImR1bW15IiwiY291bnRyeSI6IiIsImNvbXBhbnkiOiIiLCJqb2J0aXRsZSI6IiIsInN1YmlkIjoiZHVtbXlpZCIsImlkcCI6ImR1bW15IiwiaGQiOiJkdW1teS5jb20iLCJkZXNpZCI6ImR1bW15aWQiLCJjb250YWN0X2VtYWlsIjoiZHVtbXlAZHVtbXkuY29tIiwianRpIjoiNGEyMWYyYzItZjU5Yy00NWZhLTk0MTAtNDNkNDdhMTg4ODgwIn0.nkiyKtfXXxAlC60iDjXuB2EAGDfZiVglP-CyU1T4etc"
if __name__ == '__main__':
get_id_token()
\ No newline at end of file
......@@ -83,6 +83,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Legal tag of the dataset."
in: header
name: ltag
......@@ -142,6 +148,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -188,6 +200,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -237,6 +255,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -300,6 +324,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -358,6 +388,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -404,6 +440,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -446,6 +488,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -496,6 +544,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -548,6 +602,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "The tenant project name."
in: path
name: tenantid
......@@ -579,6 +639,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -596,7 +662,14 @@ paths:
items:
type: string
collectionFormat: multi
- description: 'Limit the number of datasets in the response'
in: query
name: limit
type: string
- description: 'Cursor for pagination on the datasets list'
in: query
name: cursor
type: string
responses:
200:
description: "The list of all datasets in the subproject if no gtags are in the request parameters. If gtags exist in the request parameters, then list all datasets that have the same list of gtags."
......@@ -604,6 +677,10 @@ paths:
type: array
items:
$ref: "#/definitions/Dataset"
201:
description: "Paginated dataset list with nextPageCursor. For documentation purposes, if limit or cursor is given, status code here is 200."
schema:
$ref: "#/definitions/PaginatedDatasets"
400:
description: "Bad request."
401:
......@@ -623,6 +700,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -666,6 +749,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -709,6 +798,12 @@ paths:
tags:
- Utility
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Seismic store path, sd://tenant/sub-project/path."
in: query
name: sdpath
......@@ -758,6 +853,12 @@ paths:
tags:
- Utility
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Seismic store source dataset path."
in: query
name: sdpath_from
......@@ -803,6 +904,12 @@ paths:
tags:
- Utility
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Seismic store path in the format sd://tenant/sub-project."
in: query
name: sdpath
......@@ -842,7 +949,7 @@ paths:
name: request body
required: true
schema:
$ref: "#/definitions/ImpersonationToken"
$ref: "#/definitions/ImpTokenRequest"
responses:
200:
description: "Generated a impersonation credentials token successfully."
......@@ -961,9 +1068,9 @@ paths:
type: string
required: true
default: ""
- description: "The name of the tenant/data-partition."
in: query
name: tenant-name
- description: "The impersonation token context."
in: header
name: impersonation-token-context
type: string
required: true
default: ""
......@@ -980,72 +1087,6 @@ paths:
description: "Forbidden"
404:
description: "Not found"
delete:
summary: "Revoke the impersonation token and/or a list of impersonation token signatures."
description: "<ul><li>Revoke the impersonation token and/or a list of impersonation token signatures</li><li>Required roles: app.trusted</li></ul>"
operationId: impersonation-token-revoke
tags:
- Impersonation Token
parameters:
- description: "The impersonation token to revoke (required if the signatures body field is not specified)."
in: header
name: impersonation-token
type: string
required: false
default: ""
- description: "The name of the tenant/data-partition."
in: query
name: tenant-name
type: string
required: true
default: ""
- description: "List of impersonation token signatures."
in: query
name: signatures
required: false
type: array
items:
type: string
default: ""
collectionFormat: multi
responses:
200:
description: "The impersonation token and/or the requested signatures have been successfully revoked."
400:
description: "Bad request"
401:
description: "Unauthorized"
403:
description: "Forbidden"
404:
description: "Not found"
/api/v3/impersonation-token/signatures:
get:
summary: "Retrieve the list of active impersonation token signatures."
description: "<ul><li>Retrieve the list of active impersonation token signatures.</li><li>Required roles: app.trusted</li></ul>"
operationId: impersonation-token-signature
tags:
- Impersonation Token
parameters:
- description: "The name of the tenant/data-partition"
in: query
name: tenant-name
type: string
required: true
default: ""
responses:
200:
description: "The list of signatures with their metadata."
schema:
$ref: "#/definitions/ImpersonationTokenSignatureResponse"
400:
description: "Bad request."
401:
description: "Unauthorized."
403:
description: "Forbidden."
404:
description: "Not found."
/subproject/tenant/{tenantid}/subproject/{subprojectid}:
post:
......@@ -1851,7 +1892,7 @@ definitions:
# OK
ImpersonationToken:
required: ["impersonation_token", "token_type", "expires_in"]
required: ["impersonation_token", "token_type", "expires_in", "context"]
properties:
impersonation_token:
type: string
......@@ -1862,10 +1903,14 @@ definitions:
expires_in:
type: number
description: Token expiration time.
context:
type: string
description: the Impersonation token context.
example:
impersonation_token: "ya29.fgdgsdngevrjbinb0exdnberoibnerbnerber-fdsfwefwe_cece.rfd43f3"
token_type: "Bearer"
expires_in: 3600
context: "xf420cvrv303fm4vksvdkvnejvrjbinb0exdnberonswc2mvmalksdvdeakvwrmk"
# OK
Resource:
......@@ -1894,37 +1939,6 @@ definitions:
{ "resource": "sd://tnx01/spx02", "readonly": false }
]
metadata: { "jobId": 1234 }
# OK
ImpersonationTokenSignature:
required: ["created_by", "created_date", "resources", "signature"]
properties:
created_by:
type: string
description: The trusted app id .
created_date:
type: string
description: The create date and time.
resources:
type: array
items:
$ref: "#/definitions/Resource"
signature:
type: string
description: the impersonation token signature.
metadata:
type: object
description: the custom metadata associated.
# OK
ImpersonationTokenSignatureResponse:
required: ["signatures"]
properties:
signatures:
type: array
items:
$ref: "#/definitions/ImpersonationTokenSignature"
# OK
ImpTokenRequest:
required: ["token", "resources", "refresh-url"]
......@@ -2193,6 +2207,16 @@ definitions:
type: string
description: Next cursor for pagination.
example: { datasets: ["folderA/", "folderB/", "dataset01"], nextPageCursor: "abc1234" }
# PaginatedDatasets
PaginatedDatasets:
properties:
datasets:
type: array
items:
$ref: '#/definitions/Dataset'
nextPageCursor:
type: string
# ===========================================================================
# Endpoints Security Section
......
......@@ -26,13 +26,13 @@ info:
description: "Seismic Data Management APIs to store and manage seismic datasets ."
host: "#{gcp.endpoints.service.name}#"
basePath: "/"
basePath: "/seistore-svc"
x-google-endpoints:
- name: "#{gcp.endpoints.service.name}#"
allowCors: True
x-google-allow: all
x-google-allow: configured
consumes:
- application/json
......@@ -93,6 +93,12 @@ paths:
type: string
required: false
default: "Slb-Private-USA-EHC"