Commit 8e3f4287 authored by Diego Molteni's avatar Diego Molteni
Browse files

Merge branch 'master' into slb/sb/fix-openapi-tenant

parents 1532571f 67e0d9e4
Pipeline #61899 failed with stages
in 16 minutes and 9 seconds
......@@ -5,12 +5,16 @@ variables:
PORT: 80
REPLICA: 1
UTEST_RUNTIME_IMAGE: seistore-svc-runtime
SDMS_MIN_REPLICAS: 1
SDMS_MAX_REPLICAS: 5
#aws variables
AWS_SERVICE: seismic-store
AWS_BUILD_SUBDIR: src/cloud/providers/aws/build-aws
AWS_TEST_SUBDIR: tests
AWS_ENVIRONMENT: dev
AWS_DEPLOY_TARGET: EKS
AWS_EKS_DEPLOYMENT_NAME: os-seismic-store
AWS_BUILDER_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/builder.Dockerfile
AWS_RUNTIME_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/runtime.Dockerfile
# skipping tests here. Using a local file to run tests
......@@ -34,7 +38,7 @@ variables:
OSDU_GCP_APPLICATION: seismic-store
OSDU_GCP_ENTITLEMENT_BASE_URL_PATH: /entitlements/v2
OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY: data-partition-id
OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE: https://os-legal-attcrcktoa-uc.a.run.app/api
OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE: https://community.osdu-gcp.go3-nrg.projects.epam.com/api
OSDU_GCP_DES_SERVICE_HOST_STORAGE: https://os-storage-attcrcktoa-uc.a.run.app/api
OSDU_GCP_ENV_VARS: CLOUDPROVIDER=${OSDU_GCP_CLOUD_PROVIDER},DES_SERVICE_HOST_PARTITION=${OSDU_GCP_PARTITION_API},ENTITLEMENT_BASE_URL_PATH=${OSDU_GCP_ENTITLEMENT_BASE_URL_PATH},DATA_PARTITION_REST_HEADER_KEY=${OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY},DES_SERVICE_HOST_STORAGE=${OSDU_GCP_DES_SERVICE_HOST_STORAGE},DES_SERVICE_HOST_COMPLIANCE=${OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE},SEISTORE_DES_TARGET_AUDIENCE=${GOOGLE_AUDIENCE},SERVICE_CLOUD_PROJECT=${OSDU_GCP_PROJECT},APP_ENVIRONMENT_IDENTIFIER=${TENANT},IMP_SERVICE_ACCOUNT_SIGNER=${OSDU_GCP_IMP_SERVICE_ACCOUNT_SIGNER},DES_SERVICE_HOST_ENTITLEMENT=${OSDU_GCP_ENTITLEMENTS_V2_BASE_URL},SEISTORE_DES_APPKEY=${OSDU_GCP_SEISTORE_DES_APPKEY},DES_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_DES_REDIS_INSTANCE_ADDRESS},DES_REDIS_INSTANCE_PORT=${OSDU_GCP_DES_REDIS_INSTANCE_PORT},LOCKSMAP_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_LOCKSMAP_REDIS_INSTANCE_ADDRESS} --vpc-connector=$OSDU_GCP_VPC_CONNECTOR
......@@ -52,6 +56,9 @@ include:
# lint
- local: "/devops/osdu/scanners/lint-node.yml"
# scan for secrets
- local: "/devops/osdu/scanners/scan-for-secrets-node.yml"
# containerize
- project: "osdu/platform/ci-cd-pipelines"
......@@ -91,3 +98,20 @@ osdu-gcp-test-python:
only:
variables:
- $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE == 'python'
osdu-gcp-containerize-gitlab:
stage: containerize
needs: ["compile-and-unit-test"]
tags: ["osdu-medium"]
extends: .osdu-gcp-variables
image: docker:19.03
cache: {}
allow_failure: true
script:
- export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:latest" ; fi
- docker build -t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --file docker/runtime.Dockerfile .
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- docker push $CI_REGISTRY_IMAGE/osdu-gcp
only:
variables:
- $OSDU_GCP == 'true'
This diff is collapsed.
aws-test-newman:
extends:
- .aws
- .aws_base_variables
- .aws_common_variables
- .aws_variables
stage: integration
image: node
needs: ['aws-update-ecs']
image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-node/aws-node:v1.0-node14
needs: [{ job: 'aws-update-ecs', optional: true }, { job: 'aws-update-eks', optional: true }]
script:
- apt-get update
- apt-get install -y python
- apt-get install -y python-pip
- pip install -r devops/aws/requirements.txt
- svctoken=$(python devops/scripts/aws_jwt_client.py)
- pip3 install -r devops/aws/requirements.txt
- svctoken=$(python3 devops/scripts/aws_jwt_client.py)
- npm install -g newman
- chmod +x ./tests/e2e/run_e2e_tests.sh
- ./tests/e2e/run_e2e_tests.sh --seistore-svc-url=${SEISMICSTORE_SVC_URL} --seistore-svc-api-key="NA" --user-idtoken="$svctoken" --user-idtoken="$svctoken" --tenant=opendes --subproject=awsdemosubproject --admin-email="${AWS_COGNITO_AUTH_PARAMS_USER}" --datapartition=opendes --legaltag01=opendes-sdmstestlegaltag --legaltag02=opendes-sdmstestlegaltag --subproject-long-name=subprojectlonggggggggggggggggggggggname --VCS-Provider="${ISGITLAB}"
......
......@@ -2,6 +2,11 @@ global:
replicaCount: #{REPLICA_COUNT}#
namespace: osdu
podidentity: osdu-identity
nodepool: services
isAutoscalingEnabled: false
minReplicaCount: #{SDMS_MIN_REPLICAS}#
maxReplicaCount: #{SDMS_MAX_REPLICAS}#
configEnv:
cloudProvider: #{PROVIDER_NAME}#
......@@ -15,4 +20,4 @@ configEnv:
image:
repository: #{CONTAINER_REGISTRY_NAME}#
branch: master
tag: #{IMAGE_TAG}#
\ No newline at end of file
tag: #{IMAGE_TAG}#
......@@ -15,7 +15,11 @@ spec:
labels:
app: {{ .Release.Name }}
aadpodidbinding: {{ .Values.global.podidentity }}
spec:
spec:
{{- if .Values.global.isAutoscalingEnabled }}
nodeSelector:
nodepool: {{ .Values.global.nodepool }}
{{- end }}
containers:
- name: {{ .Release.Name }}
image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
......
apiVersion: autoscaling/v2beta2
kind: HorizontalPodAutoscaler
metadata:
name: {{ .Release.Name }}
namespace: osdu
spec:
scaleTargetRef:
apiVersion: apps/v1
kind: Deployment
name: {{ .Release.Name }}
minReplicas: {{ .Values.global.minReplicaCount }}
maxReplicas: {{ .Values.global.maxReplicaCount }}
metrics:
- type: Resource
resource:
name: cpu
target:
type: Utilization
averageUtilization: 50
behavior:
scaleDown:
stabilizationWindowSeconds: 300
# Either remove 20% of current pods or 2 pods (whichever is lower) every 15 seconds until the the HPA stabilizes
selectPolicy: Min
policies:
- type: Percent
value: 20
periodSeconds: 15
- type: Pods
value: 2
periodSeconds: 15
scaleUp:
stabilizationWindowSeconds: 0
# Either add 100% of current pods or 4 pods (whichever is higher) every 1 second until the the HPA stabilizes
selectPolicy: Max
policies:
- type: Percent
value: 85
periodSeconds: 1
- type: Pods
value: 4
periodSeconds: 1
\ No newline at end of file
......@@ -12,5 +12,6 @@ spec:
- protocol: TCP
port: 80
targetPort: 80
name: http
selector:
app: {{ .Release.Name }}
\ No newline at end of file
app: {{ .Release.Name }}
......@@ -117,4 +117,12 @@ $ docker run --rm -it -v $(pwd):/opt community.opengroup.org:5555/osdu/platform/
```bash
$ docker run --rm -it -v $(pwd):/opt community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service/seismic-store-service-detect-secrets:latest detect-secrets-hook --baseline /opt/devops/docker/detect_secrets/.secrets.baseline $(git ls-files)
```
##### False positives
Add next comment above the line (in the proper file) that has been detected and is a false positives
```
pragma: allowlist nextline secret
```
\ No newline at end of file
......@@ -39,6 +39,8 @@ azure_deploy:
- sed -i 's/#{REDIS_HOST}#/'$REDIS_INSTANCE_ADDRESS'/' ${CHART_PATH}/values.yaml
- sed -i 's/#{REDIS_PORT}#/'$REDIS_INSTANCE_PORT'/' ${CHART_PATH}/values.yaml
- sed -i 's/#{REPLICA_COUNT}#/'$REPLICA'/' ${CHART_PATH}/values.yaml
- sed -i 's/#{SDMS_MIN_REPLICAS}#/'$SDMS_MIN_REPLICAS'/' ${CHART_PATH}/values.yaml
- sed -i 's/#{SDMS_MAX_REPLICAS}#/'$SDMS_MAX_REPLICAS'/' ${CHART_PATH}/values.yaml
# Install helm chart
- helm upgrade $SERVICE_NAME ${CHART_PATH} --install --dry-run --values $CHART_PATH/values.yaml
- helm upgrade $SERVICE_NAME ${CHART_PATH} --install --values $CHART_PATH/values.yaml
......
......@@ -2,6 +2,8 @@ scan-for-secrets:
image: community.opengroup.org:5555/osdu/platform/domain-data-mgmt-services/seismic/seismic-dms-suite/seismic-store-service/seismic-store-service-detect-secrets:latest
tags: ["osdu-small"]
stage: scan
needs: ['compile-and-unit-test']
needs:
- job: compile-and-unit-test
artifacts: false
script:
- detect-secrets-hook --exclude-files devops/docker/detect_secrets/.secrets.baseline --exclude-files devops/osdu/scanners/scan-for-secrets-node.yml --exclude-files npm-shrinkwrap.json --exclude-files package.json --exclude-files devops/scripts/azure_jwt_client.py --exclude-files src/cloud/providers/azure/keyvault.ts --exclude-files tests/utest/cloud/azure/keyvault.ts --baseline devops/docker/detect_secrets/.secrets.baseline $(git ls-files)
\ No newline at end of file
- detect-secrets-hook --exclude-files npm-shrinkwrap.json --exclude-files package.json --exclude-files devops/osdu/scanners/scan-for-secrets-node.yml --baseline devops/docker/detect_secrets/.secrets.baseline $(git ls-files)
\ No newline at end of file
......@@ -50,7 +50,8 @@ def get_invalid_token():
}
'''
return "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJkdW1teUBkdW1teS5jb20iLCJpc3MiOiJkdW1teUBkdW1teS5jb20iLCJhdWQiOiJkdW1teS5kdW1teS5jb20iLCJpYXQiOjE1NTYxMzcyNzMsImV4cCI6MTU1NjIzMDk3OSwicHJvdmlkZXIiOiJkdW1teS5jb20iLCJjbGllbnQiOiJkdW1teS5jb20iLCJ1c2VyaWQiOiJkdW1teXRlc3Rlci5jb20iLCJlbWFpbCI6ImR1bW15dGVzdGVyLmNvbSIsImF1dGh6IjoiIiwibGFzdG5hbWUiOiJkdW1teSIsImZpcnN0bmFtZSI6ImR1bW15IiwiY291bnRyeSI6IiIsImNvbXBhbnkiOiIiLCJqb2J0aXRsZSI6IiIsInN1YmlkIjoiZHVtbXlpZCIsImlkcCI6ImR1bW15IiwiaGQiOiJkdW1teS5jb20iLCJkZXNpZCI6ImR1bW15aWQiLCJjb250YWN0X2VtYWlsIjoiZHVtbXlAZHVtbXkuY29tIiwianRpIjoiNGEyMWYyYzItZjU5Yy00NWZhLTk0MTAtNDNkNDdhMTg4ODgwIn0.nkiyKtfXXxAlC60iDjXuB2EAGDfZiVglP-CyU1T4etc"
# pragma: allowlist nextline secret
return "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJkdW1teUBkdW1teS5jb20iLCJpc3MiOiJkdW1teUBkdW1teS5jb20iLCJhdWQiOiJkdW1teS5kdW1teS5jb20iLCJpYXQiOjE1NTYxMzcyNzMsImV4cCI6MTU1NjIzMDk3OSwicHJvdmlkZXIiOiJkdW1teS5jb20iLCJjbGllbnQiOiJkdW1teS5jb20iLCJ1c2VyaWQiOiJkdW1teXRlc3Rlci5jb20iLCJlbWFpbCI6ImR1bW15dGVzdGVyLmNvbSIsImF1dGh6IjoiIiwibGFzdG5hbWUiOiJkdW1teSIsImZpcnN0bmFtZSI6ImR1bW15IiwiY291bnRyeSI6IiIsImNvbXBhbnkiOiIiLCJqb2J0aXRsZSI6IiIsInN1YmlkIjoiZHVtbXlpZCIsImlkcCI6ImR1bW15IiwiaGQiOiJkdW1teS5jb20iLCJkZXNpZCI6ImR1bW15aWQiLCJjb250YWN0X2VtYWlsIjoiZHVtbXlAZHVtbXkuY29tIiwianRpIjoiNGEyMWYyYzItZjU5Yy00NWZhLTk0MTAtNDNkNDdhMTg4ODgwIn0.nkiyKtfXXxAlC60iDjXuB2EAGDfZiVglP-CyU1T4etc"
if __name__ == '__main__':
get_id_token()
\ No newline at end of file
......@@ -83,6 +83,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Legal tag of the dataset."
in: header
name: ltag
......@@ -142,6 +148,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -188,6 +200,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -237,6 +255,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -300,6 +324,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -358,6 +388,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -404,6 +440,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -446,6 +488,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -496,6 +544,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -548,6 +602,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "The tenant project name."
in: path
name: tenantid
......@@ -579,6 +639,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -596,7 +662,14 @@ paths:
items:
type: string
collectionFormat: multi
- description: 'Limit the number of datasets in the response'
in: query
name: limit
type: string
- description: 'Cursor for pagination on the datasets list'
in: query
name: cursor
type: string
responses:
200:
description: "The list of all datasets in the subproject if no gtags are in the request parameters. If gtags exist in the request parameters, then list all datasets that have the same list of gtags."
......@@ -604,6 +677,10 @@ paths:
type: array
items:
$ref: "#/definitions/Dataset"
201:
description: "Paginated dataset list with nextPageCursor. For documentation purposes, if limit or cursor is given, status code here is 200."
schema:
$ref: "#/definitions/PaginatedDatasets"
400:
description: "Bad request."
401:
......@@ -623,6 +700,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -666,6 +749,12 @@ paths:
tags:
- Dataset
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Name of the tenant."
in: path
name: tenantid
......@@ -709,6 +798,12 @@ paths:
tags:
- Utility
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Seismic store path, sd://tenant/sub-project/path."
in: query
name: sdpath
......@@ -758,6 +853,12 @@ paths:
tags:
- Utility
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Seismic store source dataset path."
in: query
name: sdpath_from
......@@ -803,6 +904,12 @@ paths:
tags:
- Utility
parameters:
- description: "The impersonation token context (required only with impersonation token credentials)"
in: header
name: impersonation-token-context
type: string
required: false
default: ""
- description: "Seismic store path in the format sd://tenant/sub-project."
in: query
name: sdpath
......@@ -842,7 +949,7 @@ paths:
name: request body
required: true
schema:
$ref: "#/definitions/ImpersonationToken"
$ref: "#/definitions/ImpTokenRequest"
responses:
200:
description: "Generated a impersonation credentials token successfully."
......@@ -961,9 +1068,9 @@ paths:
type: string
required: true
default: ""
- description: "The name of the tenant/data-partition."
in: query
name: tenant-name
- description: "The impersonation token context."
in: header
name: impersonation-token-context
type: string
required: true
default: ""
......@@ -980,72 +1087,6 @@ paths:
description: "Forbidden"
404:
description: "Not found"
delete:
summary: "Revoke the impersonation token and/or a list of impersonation token signatures."
description: "<ul><li>Revoke the impersonation token and/or a list of impersonation token signatures</li><li>Required roles: app.trusted</li></ul>"
operationId: impersonation-token-revoke
tags:
- Impersonation Token
parameters:
- description: "The impersonation token to revoke (required if the signatures body field is not specified)."
in: header
name: impersonation-token
type: string
required: false
default: ""
- description: "The name of the tenant/data-partition."
in: query
name: tenant-name
type: string
required: true
default: ""
- description: "List of impersonation token signatures."
in: query
name: signatures
required: false
type: array
items:
type: string
default: ""
collectionFormat: multi
responses:
200:
description: "The impersonation token and/or the requested signatures have been successfully revoked."
400:
description: "Bad request"
401:
description: "Unauthorized"
403:
description: "Forbidden"
404:
description: "Not found"
/api/v3/impersonation-token/signatures:
get:
summary: "Retrieve the list of active impersonation token signatures."
description: "<ul><li>Retrieve the list of active impersonation token signatures.</li><li>Required roles: app.trusted</li></ul>"
operationId: impersonation-token-signature
tags:
- Impersonation Token
parameters:
- description: "The name of the tenant/data-partition"
in: query
name: tenant-name
type: string
required: true
default: ""
responses:
200:
description: "The list of signatures with their metadata."
schema:
$ref: "#/definitions/ImpersonationTokenSignatureResponse"
400:
description: "Bad request."
401:
description: "Unauthorized."
403:
description: "Forbidden."
404:
description: "Not found."
/subproject/tenant/{tenantid}/subproject/{subprojectid}:
post:
......@@ -1851,7 +1892,7 @@ definitions:
# OK
ImpersonationToken:
required: ["impersonation_token", "token_type", "expires_in"]
required: ["impersonation_token", "token_type", "expires_in", "context"]
properties:
impersonation_token:
type: string
......@@ -1862,10 +1903,14 @@ definitions:
expires_in:
type: number
description: Token expiration time.
context:
type: string
description: the Impersonation token context.
example:
impersonation_token: "ya29.fgdgsdngevrjbinb0exdnberoibnerbnerber-fdsfwefwe_cece.rfd43f3"
token_type: "Bearer"
expires_in: 3600
context: "xf420cvrv303fm4vksvdkvnejvrjbinb0exdnberonswc2mvmalksdvdeakvwrmk"
# OK
Resource:
......@@ -1894,37 +1939,6 @@ definitions:
{ "resource": "sd://tnx01/spx02", "readonly": false }
]
metadata: { "jobId": 1234 }
# OK
ImpersonationTokenSignature:
required: ["created_by", "created_date", "resources", "signature"]
properties:
created_by:
type: string
description: The trusted app id .
created_date:
type: string
description: The create date and time.
resources:
type: array
items:
$ref: "#/definitions/Resource"
signature:
type: string
description: the impersonation token signature.
metadata:
type: object
description: the custom metadata associated.
# OK
ImpersonationTokenSignatureResponse:
required: ["signatures"]
properties:
signatures:
type: array
items:
$ref: "#/definitions/ImpersonationTokenSignature"
# OK
ImpTokenRequest:
required: ["token", "resources", "refresh-url"]
......@@ -2200,6 +2214,16 @@ definitions:
type: string
description: Next cursor for pagination.
example: { datasets: ["folderA/", "folderB/", "dataset01"], nextPageCursor: "abc1234" }
# PaginatedDatasets
PaginatedDatasets:
properties:
datasets: