From e59d58093e9668828af795867dd131342c522e58 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 08:34:39 -0500 Subject: [PATCH 01/45] ci: changing repo structure --- .gitlab-ci.yml | 115 +----------------- .../sdms/.devcontainer}/Dockerfile | 0 .../sdms/.devcontainer}/README.md | 0 .../sdms/.devcontainer}/bootstrap.sh | 0 .../sdms/.devcontainer}/devcontainer.json | 0 .../sdms/.devcontainer}/readme/bash.png | Bin .../.devcontainer}/readme/command-palette.png | Bin .../sdms/.devcontainer}/readme/container.png | Bin .dockerignore => app/sdms/.dockerignore | 0 .fossa.yml => app/sdms/.fossa.yml | 0 .gitattributes => app/sdms/.gitattributes | 0 .gitignore => app/sdms/.gitignore | 0 app/sdms/.gitlab-ci.yml | 48 ++++++++ {.husky => app/sdms/.husky}/pre-commit | 0 .mocharc.json => app/sdms/.mocharc.json | 0 CHANGELOG.md => app/sdms/CHANGELOG.md | 0 LICENSE => app/sdms/LICENSE | 0 NOTICE => app/sdms/NOTICE | 0 app/sdms/README.md | 54 ++++++++ {devops => app/sdms/devops}/aws/awstest.yml | 0 .../sdms/devops}/aws/requirements.txt | 0 .../sdms/devops}/azure/chart/Chart.yaml | 0 .../azure/chart/helm-config-rollback.yaml | 0 .../sdms/devops}/azure/chart/helm-config.yaml | 0 .../templates/azure-istio-auth-policy.yaml | 0 .../azure/chart/templates/configmap.yaml | 0 .../azure/chart/templates/deployment.yaml | 0 .../devops}/azure/chart/templates/hpa.yaml | 0 .../azure/chart/templates/service.yaml | 0 .../devops}/azure/development-pipeline.yml | 0 .../sdms/devops}/azure/pipeline.yml | 0 .../devops}/azure/template/build-stage.yml | 0 .../devops}/azure/template/deploy-stage.yml | 0 .../template/task/aks-deployment-steps.yml | 0 .../template/task/build-and-unit-test.yml | 0 .../template/task/build-runtime-image.yml | 0 .../azure/template/task/build-steps.yml | 0 .../task/copy-code-coverage-report.yml | 0 .../template/task/deployment-helm-steps.yml | 0 .../template/task/deployment-image-steps.yml | 0 .../template/task/download-build-artifact.yml | 0 .../devops}/azure/template/task/e2e-steps.yml | 0 .../sdms/devops}/azure/template/task/e2e.yml | 0 .../azure/template/task/flux-service-wait.yml | 0 .../azure/template/task/generate-token.yml | 0 .../devops}/azure/template/task/gitops.yml | 0 .../azure/template/task/helm-template.yml | 0 .../task/install-e2e-requirements.yml | 0 .../sdms/devops}/azure/template/task/kube.yml | 0 .../template/task/publish-build-artifact.yml | 0 .../template/task/publish-code-coverage.yml | 0 .../task/publish-unit-test-results.yml | 0 .../template/task/push-runtime-image.yml | 0 .../azure/template/task/rollback-steps.yml | 0 .../devops}/azure/template/task/rollback.yml | 0 .../devops}/azure/template/task/run-e2e.yml | 0 .../docker/detect_secrets/.secrets.baseline | 0 .../devops}/docker/detect_secrets/Dockerfile | 0 .../devops}/docker/detect_secrets/README.md | 0 .../osdu/build/seismic-store-service.yml | 3 + .../azure-seismic-store-service.yml | 0 .../sdms/devops}/osdu/scanners/fossa-node.yml | 0 .../sdms/devops}/osdu/scanners/lint-node.yml | 0 .../osdu/scanners/scan-for-secrets-node.yml | 0 .../sdms/devops}/scripts/aws_jwt_client.py | 0 .../sdms/devops}/scripts/azure_jwt_client.py | 0 {devops => app/sdms/devops}/scripts/cssfix.js | 0 .../sdms/devops}/scripts/genchangelog.sh | 0 .../sdms/devops}/scripts/ibm_jwt_client.py | 0 .../sdms/devops}/scripts/utest_build_run.sh | 0 {docker => app/sdms/docker}/README.md | 0 .../sdms/docker}/builder.Dockerfile | 0 .../sdms/docker}/runtime.Dockerfile | 0 {docs => app/sdms/docs}/api/openapi.osdu.yaml | 0 {docs => app/sdms/docs}/api/openapi.yaml | 0 .../docs}/sdms-architecture-diagram-plain.png | Bin .../sdms/docs}/sdms-architecture-diagram.png | Bin .../sdms/docs}/templates/.env-sample-aws | 0 .../sdms/docs}/templates/.env-sample-azure | 0 .../sdms/docs}/templates/.env-sample-google | 0 .../sdms/npm-shrinkwrap.json | 0 package.json => app/sdms/package.json | 0 .../cloudbuild/cloudbuild.yaml | 0 {src => app/sdms/src}/auth/auth.ts | 0 {src => app/sdms/src}/auth/groups.ts | 0 {src => app/sdms/src}/auth/index.ts | 0 .../sdms/src}/auth/providers/generic/auth.ts | 0 .../sdms/src}/auth/providers/generic/index.ts | 0 {src => app/sdms/src}/auth/providers/index.ts | 0 {src => app/sdms/src}/auth/roles.ts | 0 {src => app/sdms/src}/cloud/cloud.ts | 0 {src => app/sdms/src}/cloud/config.ts | 0 {src => app/sdms/src}/cloud/credentials.ts | 0 {src => app/sdms/src}/cloud/dataecosystem.ts | 0 {src => app/sdms/src}/cloud/index.ts | 0 {src => app/sdms/src}/cloud/journal.ts | 0 {src => app/sdms/src}/cloud/logger.ts | 0 .../providers/aws/build-aws/build-info.py | 0 .../aws/build-aws/builder.Dockerfile | 0 .../providers/aws/build-aws/buildspec.yaml | 0 .../providers/aws/build-aws/entrypoint.sh | 0 .../aws/build-aws/runtime.Dockerfile | 0 .../src}/cloud/providers/aws/build-aws/ssl.sh | 0 .../sdms/src}/cloud/providers/aws/config.ts | 0 .../src}/cloud/providers/aws/credentials.ts | 0 .../src}/cloud/providers/aws/dataecosystem.ts | 0 .../sdms/src}/cloud/providers/aws/dynamodb.ts | 0 .../sdms/src}/cloud/providers/aws/index.ts | 0 .../sdms/src}/cloud/providers/aws/logger.ts | 0 .../sdms/src}/cloud/providers/aws/seistore.ts | 0 .../src}/cloud/providers/aws/ssmhelper.ts | 0 .../sdms/src}/cloud/providers/aws/storage.ts | 0 .../src}/cloud/providers/aws/stshelper.ts | 0 .../sdms/src}/cloud/providers/aws/trace.ts | 0 .../cloud/providers/azure/cloudstorage.ts | 0 .../sdms/src}/cloud/providers/azure/config.ts | 0 .../src}/cloud/providers/azure/cosmosdb.ts | 0 .../src}/cloud/providers/azure/credentials.ts | 0 .../cloud/providers/azure/dataecosystem.ts | 0 .../sdms/src}/cloud/providers/azure/index.ts | 0 .../src}/cloud/providers/azure/insights.ts | 0 .../src}/cloud/providers/azure/keyvault.ts | 0 .../src}/cloud/providers/azure/seistore.ts | 0 .../sdms/src}/cloud/providers/azure/trace.ts | 0 .../src}/cloud/providers/google/config.ts | 0 .../cloud/providers/google/credentials.ts | 0 .../cloud/providers/google/dataecosystem.ts | 0 .../src}/cloud/providers/google/datastore.ts | 0 .../sdms/src}/cloud/providers/google/gcs.ts | 0 .../sdms/src}/cloud/providers/google/index.ts | 0 .../src}/cloud/providers/google/logger.ts | 0 .../src}/cloud/providers/google/secrets.ts | 0 .../src}/cloud/providers/google/seistore.ts | 0 .../sdms/src}/cloud/providers/google/trace.ts | 0 .../sdms/src}/cloud/providers/ibm/config.ts | 0 .../sdms/src}/cloud/providers/ibm/cos.ts | 0 .../src}/cloud/providers/ibm/credentials.ts | 0 .../src}/cloud/providers/ibm/dataecosystem.ts | 0 .../src}/cloud/providers/ibm/datastore.ts | 0 .../sdms/src}/cloud/providers/ibm/index.ts | 0 .../sdms/src}/cloud/providers/ibm/logger.ts | 0 .../sdms/src}/cloud/providers/ibm/seistore.ts | 0 .../src}/cloud/providers/ibm/stshelper.ts | 0 .../sdms/src}/cloud/providers/ibm/trace.ts | 0 .../sdms/src}/cloud/providers/index.ts | 0 {src => app/sdms/src}/cloud/seistore.ts | 0 {src => app/sdms/src}/cloud/shared/queue.ts | 0 {src => app/sdms/src}/cloud/storage.ts | 0 {src => app/sdms/src}/cloud/trace.ts | 0 .../sdms/src}/dataecosystem/compliance.ts | 0 .../sdms/src}/dataecosystem/entitlement.ts | 0 {src => app/sdms/src}/dataecosystem/index.ts | 0 .../sdms/src}/dataecosystem/storage.ts | 0 {src => app/sdms/src}/dataecosystem/utils.ts | 0 {src => app/sdms/src}/metrics/index.ts | 0 {src => app/sdms/src}/metrics/metrics.ts | 0 {src => app/sdms/src}/server/index.ts | 0 {src => app/sdms/src}/server/server-start.ts | 0 {src => app/sdms/src}/server/server.ts | 0 {src => app/sdms/src}/services/dataset/dao.ts | 0 .../sdms/src}/services/dataset/handler.ts | 0 .../sdms/src}/services/dataset/index.ts | 0 .../sdms/src}/services/dataset/locker.ts | 0 .../sdms/src}/services/dataset/model.ts | 0 .../sdms/src}/services/dataset/optype.ts | 0 .../sdms/src}/services/dataset/parser.ts | 0 .../sdms/src}/services/dataset/service.ts | 0 .../sdms/src}/services/general/handler.ts | 0 .../sdms/src}/services/general/optype.ts | 0 .../sdms/src}/services/general/service.ts | 0 .../services/impersonation_token/handler.ts | 0 .../services/impersonation_token/model.ts | 0 .../services/impersonation_token/optype.ts | 0 .../services/impersonation_token/parser.ts | 0 .../services/impersonation_token/service.ts | 0 .../sdms/src}/services/imptoken/dao.ts | 0 .../sdms/src}/services/imptoken/handler.ts | 0 .../sdms/src}/services/imptoken/index.ts | 0 .../sdms/src}/services/imptoken/model.ts | 0 .../sdms/src}/services/imptoken/optype.ts | 0 .../sdms/src}/services/imptoken/parser.ts | 0 .../sdms/src}/services/imptoken/service.ts | 0 {src => app/sdms/src}/services/index.ts | 0 .../sdms/src}/services/subproject/dao.ts | 0 .../sdms/src}/services/subproject/groups.ts | 0 .../sdms/src}/services/subproject/handler.ts | 0 .../sdms/src}/services/subproject/index.ts | 0 .../sdms/src}/services/subproject/model.ts | 0 .../sdms/src}/services/subproject/optype.ts | 0 .../sdms/src}/services/subproject/parser.ts | 0 .../sdms/src}/services/subproject/service.ts | 0 {src => app/sdms/src}/services/svcapp/dao.ts | 0 .../sdms/src}/services/svcapp/handler.ts | 0 .../sdms/src}/services/svcapp/model.ts | 0 .../sdms/src}/services/svcapp/optype.ts | 0 .../sdms/src}/services/svcapp/parser.ts | 0 .../sdms/src}/services/svcapp/service.ts | 0 {src => app/sdms/src}/services/tenant/dao.ts | 0 .../sdms/src}/services/tenant/groups.ts | 0 .../sdms/src}/services/tenant/handler.ts | 0 .../sdms/src}/services/tenant/index.ts | 0 .../sdms/src}/services/tenant/model.ts | 0 .../sdms/src}/services/tenant/optype.ts | 0 .../sdms/src}/services/tenant/parser.ts | 0 .../sdms/src}/services/tenant/service.ts | 0 .../sdms/src}/services/user/handler.ts | 0 {src => app/sdms/src}/services/user/optype.ts | 0 {src => app/sdms/src}/services/user/parser.ts | 0 .../sdms/src}/services/user/service.ts | 0 .../sdms/src}/services/utility/handler.ts | 0 .../sdms/src}/services/utility/optype.ts | 0 .../sdms/src}/services/utility/parser.ts | 0 .../sdms/src}/services/utility/service.ts | 0 {src => app/sdms/src}/shared/cache.ts | 0 {src => app/sdms/src}/shared/error.ts | 0 {src => app/sdms/src}/shared/featureflags.ts | 0 {src => app/sdms/src}/shared/index.ts | 0 {src => app/sdms/src}/shared/params.ts | 0 {src => app/sdms/src}/shared/response.ts | 0 {src => app/sdms/src}/shared/sdpath.ts | 0 {src => app/sdms/src}/shared/tracelog.ts | 0 {src => app/sdms/src}/shared/utils.ts | 0 .../tests}/aws-test/build-aws/prepare-dist.sh | 0 .../tests}/aws-test/build-aws/run-tests.sh | 0 .../postman_collection.dataset_delete.json | 0 .../postman_collection.dataset_readlock.json | 0 .../postman_collection.dataset_register.json | 0 .../postman_collection.dataset_unlock.json | 0 .../postman_collection.dataset_writelock.json | 0 .../parallel/postman_collection.gcstoken.json | 0 .../sdms/tests}/e2e/parallel/postman_env.json | 0 .../e2e/parallel/run_e2e_tests_parallel.js | 0 .../e2e/parallel/run_e2e_tests_parallel.sh | 0 .../sdms/tests}/e2e/postman_collection.json | 0 .../sdms/tests}/e2e/postman_env.json | 0 .../sdms/tests}/e2e/run_e2e_tests.sh | 0 {tests => app/sdms/tests}/utest/auth/auth.ts | 0 .../sdms/tests}/utest/auth/groups.ts | 0 {tests => app/sdms/tests}/utest/auth/test.ts | 0 .../tests}/utest/cloud/azure/cloudstorage.ts | 0 .../utest/cloud/azure/cosmosdb-query.ts | 0 .../cloud/azure/cosmosdb-transactions.ts | 0 .../sdms/tests}/utest/cloud/azure/cosmosdb.ts | 0 .../sdms/tests}/utest/cloud/azure/keyvault.ts | 0 .../sdms/tests}/utest/cloud/credentials.ts | 0 .../sdms/tests}/utest/cloud/datastore.ts | 0 {tests => app/sdms/tests}/utest/cloud/gcs.ts | 0 {tests => app/sdms/tests}/utest/cloud/test.ts | 0 .../sdms/tests}/utest/dao/dataset.ts | 0 .../sdms/tests}/utest/dao/imptoken.ts | 0 .../sdms/tests}/utest/dao/subproject.ts | 0 {tests => app/sdms/tests}/utest/dao/svcapp.ts | 0 {tests => app/sdms/tests}/utest/dao/tenant.ts | 0 {tests => app/sdms/tests}/utest/dao/test.ts | 0 .../tests}/utest/dataecosystem/compliance.ts | 0 .../tests}/utest/dataecosystem/entitlement.ts | 0 .../tests}/utest/dataecosystem/storage.ts | 0 .../sdms/tests}/utest/dataecosystem/test.ts | 0 .../sdms/tests}/utest/dataecosystem/utils.ts | 0 .../sdms/tests}/utest/services/app.ts | 0 .../sdms/tests}/utest/services/dataset.ts | 0 .../sdms/tests}/utest/services/general.ts | 0 .../sdms/tests}/utest/services/imptoken.ts | 0 .../sdms/tests}/utest/services/locker.ts | 0 .../sdms/tests}/utest/services/subproject.ts | 0 .../sdms/tests}/utest/services/tenant.ts | 0 .../sdms/tests}/utest/services/test.ts | 0 .../sdms/tests}/utest/services/user.ts | 0 .../sdms/tests}/utest/services/utility.ts | 0 .../sdms/tests}/utest/shared/error.ts | 0 .../sdms/tests}/utest/shared/logger.ts | 0 .../sdms/tests}/utest/shared/params.ts | 0 .../sdms/tests}/utest/shared/response.ts | 0 .../sdms/tests}/utest/shared/sdpath.ts | 0 .../sdms/tests}/utest/shared/test.ts | 0 .../sdms/tests}/utest/shared/utils.ts | 0 {tests => app/sdms/tests}/utest/test.ts | 0 {tests => app/sdms/tests}/utest/utils.ts | 0 tsconfig.json => app/sdms/tsconfig.json | 0 tslint.json => app/sdms/tslint.json | 0 280 files changed, 108 insertions(+), 112 deletions(-) rename {.devcontainer => app/sdms/.devcontainer}/Dockerfile (100%) rename {.devcontainer => app/sdms/.devcontainer}/README.md (100%) rename {.devcontainer => app/sdms/.devcontainer}/bootstrap.sh (100%) rename {.devcontainer => app/sdms/.devcontainer}/devcontainer.json (100%) rename {.devcontainer => app/sdms/.devcontainer}/readme/bash.png (100%) rename {.devcontainer => app/sdms/.devcontainer}/readme/command-palette.png (100%) rename {.devcontainer => app/sdms/.devcontainer}/readme/container.png (100%) rename .dockerignore => app/sdms/.dockerignore (100%) rename .fossa.yml => app/sdms/.fossa.yml (100%) rename .gitattributes => app/sdms/.gitattributes (100%) rename .gitignore => app/sdms/.gitignore (100%) create mode 100644 app/sdms/.gitlab-ci.yml rename {.husky => app/sdms/.husky}/pre-commit (100%) rename .mocharc.json => app/sdms/.mocharc.json (100%) rename CHANGELOG.md => app/sdms/CHANGELOG.md (100%) rename LICENSE => app/sdms/LICENSE (100%) rename NOTICE => app/sdms/NOTICE (100%) create mode 100644 app/sdms/README.md rename {devops => app/sdms/devops}/aws/awstest.yml (100%) rename {devops => app/sdms/devops}/aws/requirements.txt (100%) rename {devops => app/sdms/devops}/azure/chart/Chart.yaml (100%) rename {devops => app/sdms/devops}/azure/chart/helm-config-rollback.yaml (100%) rename {devops => app/sdms/devops}/azure/chart/helm-config.yaml (100%) rename {devops => app/sdms/devops}/azure/chart/templates/azure-istio-auth-policy.yaml (100%) rename {devops => app/sdms/devops}/azure/chart/templates/configmap.yaml (100%) rename {devops => app/sdms/devops}/azure/chart/templates/deployment.yaml (100%) rename {devops => app/sdms/devops}/azure/chart/templates/hpa.yaml (100%) rename {devops => app/sdms/devops}/azure/chart/templates/service.yaml (100%) rename {devops => app/sdms/devops}/azure/development-pipeline.yml (100%) rename {devops => app/sdms/devops}/azure/pipeline.yml (100%) rename {devops => app/sdms/devops}/azure/template/build-stage.yml (100%) rename {devops => app/sdms/devops}/azure/template/deploy-stage.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/aks-deployment-steps.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/build-and-unit-test.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/build-runtime-image.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/build-steps.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/copy-code-coverage-report.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/deployment-helm-steps.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/deployment-image-steps.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/download-build-artifact.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/e2e-steps.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/e2e.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/flux-service-wait.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/generate-token.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/gitops.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/helm-template.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/install-e2e-requirements.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/kube.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/publish-build-artifact.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/publish-code-coverage.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/publish-unit-test-results.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/push-runtime-image.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/rollback-steps.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/rollback.yml (100%) rename {devops => app/sdms/devops}/azure/template/task/run-e2e.yml (100%) rename {devops => app/sdms/devops}/docker/detect_secrets/.secrets.baseline (100%) rename {devops => app/sdms/devops}/docker/detect_secrets/Dockerfile (100%) rename {devops => app/sdms/devops}/docker/detect_secrets/README.md (100%) rename {devops => app/sdms/devops}/osdu/build/seismic-store-service.yml (92%) rename {devops => app/sdms/devops}/osdu/cloud-providers/azure-seismic-store-service.yml (100%) rename {devops => app/sdms/devops}/osdu/scanners/fossa-node.yml (100%) rename {devops => app/sdms/devops}/osdu/scanners/lint-node.yml (100%) rename {devops => app/sdms/devops}/osdu/scanners/scan-for-secrets-node.yml (100%) rename {devops => app/sdms/devops}/scripts/aws_jwt_client.py (100%) rename {devops => app/sdms/devops}/scripts/azure_jwt_client.py (100%) rename {devops => app/sdms/devops}/scripts/cssfix.js (100%) rename {devops => app/sdms/devops}/scripts/genchangelog.sh (100%) rename {devops => app/sdms/devops}/scripts/ibm_jwt_client.py (100%) rename {devops => app/sdms/devops}/scripts/utest_build_run.sh (100%) rename {docker => app/sdms/docker}/README.md (100%) rename {docker => app/sdms/docker}/builder.Dockerfile (100%) rename {docker => app/sdms/docker}/runtime.Dockerfile (100%) rename {docs => app/sdms/docs}/api/openapi.osdu.yaml (100%) rename {docs => app/sdms/docs}/api/openapi.yaml (100%) rename {docs => app/sdms/docs}/sdms-architecture-diagram-plain.png (100%) rename {docs => app/sdms/docs}/sdms-architecture-diagram.png (100%) rename {docs => app/sdms/docs}/templates/.env-sample-aws (100%) rename {docs => app/sdms/docs}/templates/.env-sample-azure (100%) rename {docs => app/sdms/docs}/templates/.env-sample-google (100%) rename npm-shrinkwrap.json => app/sdms/npm-shrinkwrap.json (100%) rename package.json => app/sdms/package.json (100%) rename {provider => app/sdms/provider}/seismic-store-gcp/cloudbuild/cloudbuild.yaml (100%) rename {src => app/sdms/src}/auth/auth.ts (100%) rename {src => app/sdms/src}/auth/groups.ts (100%) rename {src => app/sdms/src}/auth/index.ts (100%) rename {src => app/sdms/src}/auth/providers/generic/auth.ts (100%) rename {src => app/sdms/src}/auth/providers/generic/index.ts (100%) rename {src => app/sdms/src}/auth/providers/index.ts (100%) rename {src => app/sdms/src}/auth/roles.ts (100%) rename {src => app/sdms/src}/cloud/cloud.ts (100%) rename {src => app/sdms/src}/cloud/config.ts (100%) rename {src => app/sdms/src}/cloud/credentials.ts (100%) rename {src => app/sdms/src}/cloud/dataecosystem.ts (100%) rename {src => app/sdms/src}/cloud/index.ts (100%) rename {src => app/sdms/src}/cloud/journal.ts (100%) rename {src => app/sdms/src}/cloud/logger.ts (100%) rename {src => app/sdms/src}/cloud/providers/aws/build-aws/build-info.py (100%) rename {src => app/sdms/src}/cloud/providers/aws/build-aws/builder.Dockerfile (100%) rename {src => app/sdms/src}/cloud/providers/aws/build-aws/buildspec.yaml (100%) rename {src => app/sdms/src}/cloud/providers/aws/build-aws/entrypoint.sh (100%) mode change 100755 => 100644 rename {src => app/sdms/src}/cloud/providers/aws/build-aws/runtime.Dockerfile (100%) rename {src => app/sdms/src}/cloud/providers/aws/build-aws/ssl.sh (100%) mode change 100755 => 100644 rename {src => app/sdms/src}/cloud/providers/aws/config.ts (100%) rename {src => app/sdms/src}/cloud/providers/aws/credentials.ts (100%) rename {src => app/sdms/src}/cloud/providers/aws/dataecosystem.ts (100%) rename {src => app/sdms/src}/cloud/providers/aws/dynamodb.ts (100%) rename {src => app/sdms/src}/cloud/providers/aws/index.ts (100%) rename {src => app/sdms/src}/cloud/providers/aws/logger.ts (100%) rename {src => app/sdms/src}/cloud/providers/aws/seistore.ts (100%) rename {src => app/sdms/src}/cloud/providers/aws/ssmhelper.ts (100%) rename {src => app/sdms/src}/cloud/providers/aws/storage.ts (100%) rename {src => app/sdms/src}/cloud/providers/aws/stshelper.ts (100%) rename {src => app/sdms/src}/cloud/providers/aws/trace.ts (100%) rename {src => app/sdms/src}/cloud/providers/azure/cloudstorage.ts (100%) rename {src => app/sdms/src}/cloud/providers/azure/config.ts (100%) rename {src => app/sdms/src}/cloud/providers/azure/cosmosdb.ts (100%) rename {src => app/sdms/src}/cloud/providers/azure/credentials.ts (100%) rename {src => app/sdms/src}/cloud/providers/azure/dataecosystem.ts (100%) rename {src => app/sdms/src}/cloud/providers/azure/index.ts (100%) rename {src => app/sdms/src}/cloud/providers/azure/insights.ts (100%) rename {src => app/sdms/src}/cloud/providers/azure/keyvault.ts (100%) rename {src => app/sdms/src}/cloud/providers/azure/seistore.ts (100%) rename {src => app/sdms/src}/cloud/providers/azure/trace.ts (100%) rename {src => app/sdms/src}/cloud/providers/google/config.ts (100%) rename {src => app/sdms/src}/cloud/providers/google/credentials.ts (100%) rename {src => app/sdms/src}/cloud/providers/google/dataecosystem.ts (100%) rename {src => app/sdms/src}/cloud/providers/google/datastore.ts (100%) rename {src => app/sdms/src}/cloud/providers/google/gcs.ts (100%) rename {src => app/sdms/src}/cloud/providers/google/index.ts (100%) rename {src => app/sdms/src}/cloud/providers/google/logger.ts (100%) rename {src => app/sdms/src}/cloud/providers/google/secrets.ts (100%) rename {src => app/sdms/src}/cloud/providers/google/seistore.ts (100%) rename {src => app/sdms/src}/cloud/providers/google/trace.ts (100%) rename {src => app/sdms/src}/cloud/providers/ibm/config.ts (100%) rename {src => app/sdms/src}/cloud/providers/ibm/cos.ts (100%) rename {src => app/sdms/src}/cloud/providers/ibm/credentials.ts (100%) rename {src => app/sdms/src}/cloud/providers/ibm/dataecosystem.ts (100%) rename {src => app/sdms/src}/cloud/providers/ibm/datastore.ts (100%) rename {src => app/sdms/src}/cloud/providers/ibm/index.ts (100%) rename {src => app/sdms/src}/cloud/providers/ibm/logger.ts (100%) rename {src => app/sdms/src}/cloud/providers/ibm/seistore.ts (100%) rename {src => app/sdms/src}/cloud/providers/ibm/stshelper.ts (100%) rename {src => app/sdms/src}/cloud/providers/ibm/trace.ts (100%) rename {src => app/sdms/src}/cloud/providers/index.ts (100%) rename {src => app/sdms/src}/cloud/seistore.ts (100%) rename {src => app/sdms/src}/cloud/shared/queue.ts (100%) rename {src => app/sdms/src}/cloud/storage.ts (100%) rename {src => app/sdms/src}/cloud/trace.ts (100%) rename {src => app/sdms/src}/dataecosystem/compliance.ts (100%) rename {src => app/sdms/src}/dataecosystem/entitlement.ts (100%) rename {src => app/sdms/src}/dataecosystem/index.ts (100%) rename {src => app/sdms/src}/dataecosystem/storage.ts (100%) rename {src => app/sdms/src}/dataecosystem/utils.ts (100%) rename {src => app/sdms/src}/metrics/index.ts (100%) rename {src => app/sdms/src}/metrics/metrics.ts (100%) rename {src => app/sdms/src}/server/index.ts (100%) rename {src => app/sdms/src}/server/server-start.ts (100%) rename {src => app/sdms/src}/server/server.ts (100%) rename {src => app/sdms/src}/services/dataset/dao.ts (100%) rename {src => app/sdms/src}/services/dataset/handler.ts (100%) rename {src => app/sdms/src}/services/dataset/index.ts (100%) rename {src => app/sdms/src}/services/dataset/locker.ts (100%) rename {src => app/sdms/src}/services/dataset/model.ts (100%) rename {src => app/sdms/src}/services/dataset/optype.ts (100%) rename {src => app/sdms/src}/services/dataset/parser.ts (100%) rename {src => app/sdms/src}/services/dataset/service.ts (100%) rename {src => app/sdms/src}/services/general/handler.ts (100%) rename {src => app/sdms/src}/services/general/optype.ts (100%) rename {src => app/sdms/src}/services/general/service.ts (100%) rename {src => app/sdms/src}/services/impersonation_token/handler.ts (100%) rename {src => app/sdms/src}/services/impersonation_token/model.ts (100%) rename {src => app/sdms/src}/services/impersonation_token/optype.ts (100%) rename {src => app/sdms/src}/services/impersonation_token/parser.ts (100%) rename {src => app/sdms/src}/services/impersonation_token/service.ts (100%) rename {src => app/sdms/src}/services/imptoken/dao.ts (100%) rename {src => app/sdms/src}/services/imptoken/handler.ts (100%) rename {src => app/sdms/src}/services/imptoken/index.ts (100%) rename {src => app/sdms/src}/services/imptoken/model.ts (100%) rename {src => app/sdms/src}/services/imptoken/optype.ts (100%) rename {src => app/sdms/src}/services/imptoken/parser.ts (100%) rename {src => app/sdms/src}/services/imptoken/service.ts (100%) rename {src => app/sdms/src}/services/index.ts (100%) rename {src => app/sdms/src}/services/subproject/dao.ts (100%) rename {src => app/sdms/src}/services/subproject/groups.ts (100%) rename {src => app/sdms/src}/services/subproject/handler.ts (100%) rename {src => app/sdms/src}/services/subproject/index.ts (100%) rename {src => app/sdms/src}/services/subproject/model.ts (100%) rename {src => app/sdms/src}/services/subproject/optype.ts (100%) rename {src => app/sdms/src}/services/subproject/parser.ts (100%) rename {src => app/sdms/src}/services/subproject/service.ts (100%) rename {src => app/sdms/src}/services/svcapp/dao.ts (100%) rename {src => app/sdms/src}/services/svcapp/handler.ts (100%) rename {src => app/sdms/src}/services/svcapp/model.ts (100%) rename {src => app/sdms/src}/services/svcapp/optype.ts (100%) rename {src => app/sdms/src}/services/svcapp/parser.ts (100%) rename {src => app/sdms/src}/services/svcapp/service.ts (100%) rename {src => app/sdms/src}/services/tenant/dao.ts (100%) rename {src => app/sdms/src}/services/tenant/groups.ts (100%) rename {src => app/sdms/src}/services/tenant/handler.ts (100%) rename {src => app/sdms/src}/services/tenant/index.ts (100%) rename {src => app/sdms/src}/services/tenant/model.ts (100%) rename {src => app/sdms/src}/services/tenant/optype.ts (100%) rename {src => app/sdms/src}/services/tenant/parser.ts (100%) rename {src => app/sdms/src}/services/tenant/service.ts (100%) rename {src => app/sdms/src}/services/user/handler.ts (100%) rename {src => app/sdms/src}/services/user/optype.ts (100%) rename {src => app/sdms/src}/services/user/parser.ts (100%) rename {src => app/sdms/src}/services/user/service.ts (100%) rename {src => app/sdms/src}/services/utility/handler.ts (100%) rename {src => app/sdms/src}/services/utility/optype.ts (100%) rename {src => app/sdms/src}/services/utility/parser.ts (100%) rename {src => app/sdms/src}/services/utility/service.ts (100%) rename {src => app/sdms/src}/shared/cache.ts (100%) rename {src => app/sdms/src}/shared/error.ts (100%) rename {src => app/sdms/src}/shared/featureflags.ts (100%) rename {src => app/sdms/src}/shared/index.ts (100%) rename {src => app/sdms/src}/shared/params.ts (100%) rename {src => app/sdms/src}/shared/response.ts (100%) rename {src => app/sdms/src}/shared/sdpath.ts (100%) rename {src => app/sdms/src}/shared/tracelog.ts (100%) rename {src => app/sdms/src}/shared/utils.ts (100%) rename {tests => app/sdms/tests}/aws-test/build-aws/prepare-dist.sh (100%) mode change 100755 => 100644 rename {tests => app/sdms/tests}/aws-test/build-aws/run-tests.sh (100%) mode change 100755 => 100644 rename {tests => app/sdms/tests}/e2e/parallel/postman_collection.dataset_delete.json (100%) rename {tests => app/sdms/tests}/e2e/parallel/postman_collection.dataset_readlock.json (100%) rename {tests => app/sdms/tests}/e2e/parallel/postman_collection.dataset_register.json (100%) rename {tests => app/sdms/tests}/e2e/parallel/postman_collection.dataset_unlock.json (100%) rename {tests => app/sdms/tests}/e2e/parallel/postman_collection.dataset_writelock.json (100%) rename {tests => app/sdms/tests}/e2e/parallel/postman_collection.gcstoken.json (100%) rename {tests => app/sdms/tests}/e2e/parallel/postman_env.json (100%) rename {tests => app/sdms/tests}/e2e/parallel/run_e2e_tests_parallel.js (100%) rename {tests => app/sdms/tests}/e2e/parallel/run_e2e_tests_parallel.sh (100%) rename {tests => app/sdms/tests}/e2e/postman_collection.json (100%) rename {tests => app/sdms/tests}/e2e/postman_env.json (100%) rename {tests => app/sdms/tests}/e2e/run_e2e_tests.sh (100%) mode change 100755 => 100644 rename {tests => app/sdms/tests}/utest/auth/auth.ts (100%) rename {tests => app/sdms/tests}/utest/auth/groups.ts (100%) rename {tests => app/sdms/tests}/utest/auth/test.ts (100%) rename {tests => app/sdms/tests}/utest/cloud/azure/cloudstorage.ts (100%) rename {tests => app/sdms/tests}/utest/cloud/azure/cosmosdb-query.ts (100%) rename {tests => app/sdms/tests}/utest/cloud/azure/cosmosdb-transactions.ts (100%) rename {tests => app/sdms/tests}/utest/cloud/azure/cosmosdb.ts (100%) rename {tests => app/sdms/tests}/utest/cloud/azure/keyvault.ts (100%) rename {tests => app/sdms/tests}/utest/cloud/credentials.ts (100%) rename {tests => app/sdms/tests}/utest/cloud/datastore.ts (100%) rename {tests => app/sdms/tests}/utest/cloud/gcs.ts (100%) rename {tests => app/sdms/tests}/utest/cloud/test.ts (100%) rename {tests => app/sdms/tests}/utest/dao/dataset.ts (100%) rename {tests => app/sdms/tests}/utest/dao/imptoken.ts (100%) rename {tests => app/sdms/tests}/utest/dao/subproject.ts (100%) rename {tests => app/sdms/tests}/utest/dao/svcapp.ts (100%) rename {tests => app/sdms/tests}/utest/dao/tenant.ts (100%) rename {tests => app/sdms/tests}/utest/dao/test.ts (100%) rename {tests => app/sdms/tests}/utest/dataecosystem/compliance.ts (100%) rename {tests => app/sdms/tests}/utest/dataecosystem/entitlement.ts (100%) rename {tests => app/sdms/tests}/utest/dataecosystem/storage.ts (100%) rename {tests => app/sdms/tests}/utest/dataecosystem/test.ts (100%) rename {tests => app/sdms/tests}/utest/dataecosystem/utils.ts (100%) rename {tests => app/sdms/tests}/utest/services/app.ts (100%) rename {tests => app/sdms/tests}/utest/services/dataset.ts (100%) rename {tests => app/sdms/tests}/utest/services/general.ts (100%) rename {tests => app/sdms/tests}/utest/services/imptoken.ts (100%) rename {tests => app/sdms/tests}/utest/services/locker.ts (100%) rename {tests => app/sdms/tests}/utest/services/subproject.ts (100%) rename {tests => app/sdms/tests}/utest/services/tenant.ts (100%) rename {tests => app/sdms/tests}/utest/services/test.ts (100%) rename {tests => app/sdms/tests}/utest/services/user.ts (100%) rename {tests => app/sdms/tests}/utest/services/utility.ts (100%) rename {tests => app/sdms/tests}/utest/shared/error.ts (100%) rename {tests => app/sdms/tests}/utest/shared/logger.ts (100%) rename {tests => app/sdms/tests}/utest/shared/params.ts (100%) rename {tests => app/sdms/tests}/utest/shared/response.ts (100%) rename {tests => app/sdms/tests}/utest/shared/sdpath.ts (100%) rename {tests => app/sdms/tests}/utest/shared/test.ts (100%) rename {tests => app/sdms/tests}/utest/shared/utils.ts (100%) rename {tests => app/sdms/tests}/utest/test.ts (100%) rename {tests => app/sdms/tests}/utest/utils.ts (100%) rename tsconfig.json => app/sdms/tsconfig.json (100%) rename tslint.json => app/sdms/tslint.json (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86c2e442..3bab8144 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,117 +1,8 @@ -variables: - #seismic store service variables - ENVIRONMENT: cloud - ISGITLAB: "true" - PORT: 80 - REPLICA: 1 - UTEST_RUNTIME_IMAGE: seistore-svc-runtime - SDMS_MIN_REPLICAS: 1 - SDMS_MAX_REPLICAS: 5 - - #aws variables - AWS_SERVICE: seismic-store - AWS_BUILD_SUBDIR: src/cloud/providers/aws/build-aws - AWS_TEST_SUBDIR: tests - AWS_ENVIRONMENT: dev - AWS_DEPLOY_TARGET: EKS - AWS_EKS_DEPLOYMENT_NAME: os-seismic-store - AWS_BUILDER_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/builder.Dockerfile - AWS_RUNTIME_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/runtime.Dockerfile - # skipping tests here. Using a local file to run tests - AWS_SKIP_TESTS: "true" - #azure variables - AZURE_SERVICE: seismic-store-service - #gcp variables - GCP_APPLICATION_NAME: os-seismic-store-service - GCP_DEPLOY_FILE: $OSDU_GCP_DEPLOY_FILE - GCP_PROJECT: opendes-evt - GCP_SERVICE: seismic-store-service - GCP_VENDOR: gcp - #ibm variables - IBM_OPENSHIFT_PROJECT: og-cicd - IBM_VENDOR: ibm - - OSDU_GCP_LOG_LEVEL: INFO - OSDU_GCP_SERVICE: seismic-store - OSDU_GCP_VENDOR: gcp - OSDU_GCP_APPLICATION_NAME: seismic-store - OSDU_GCP_APPLICATION: seismic-store - OSDU_GCP_ENTITLEMENT_BASE_URL_PATH: /entitlements/v2 - OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY: data-partition-id - OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE: https://community.osdu-gcp.go3-nrg.projects.epam.com/api - OSDU_GCP_DES_SERVICE_HOST_STORAGE: https://os-storage-attcrcktoa-uc.a.run.app/api - OSDU_GCP_ENV_VARS: CLOUDPROVIDER=${OSDU_GCP_CLOUD_PROVIDER},DES_SERVICE_HOST_PARTITION=${OSDU_GCP_PARTITION_API},ENTITLEMENT_BASE_URL_PATH=${OSDU_GCP_ENTITLEMENT_BASE_URL_PATH},DATA_PARTITION_REST_HEADER_KEY=${OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY},DES_SERVICE_HOST_STORAGE=${OSDU_GCP_DES_SERVICE_HOST_STORAGE},DES_SERVICE_HOST_COMPLIANCE=${OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE},SEISTORE_DES_TARGET_AUDIENCE=${GOOGLE_AUDIENCE},SERVICE_CLOUD_PROJECT=${OSDU_GCP_PROJECT},APP_ENVIRONMENT_IDENTIFIER=${TENANT},IMP_SERVICE_ACCOUNT_SIGNER=${OSDU_GCP_IMP_SERVICE_ACCOUNT_SIGNER},DES_SERVICE_HOST_ENTITLEMENT=${OSDU_GCP_ENTITLEMENTS_V2_BASE_URL},SEISTORE_DES_APPKEY=${OSDU_GCP_SEISTORE_DES_APPKEY},DES_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_DES_REDIS_INSTANCE_ADDRESS},DES_REDIS_INSTANCE_PORT=${OSDU_GCP_DES_REDIS_INSTANCE_PORT},LOCKSMAP_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_LOCKSMAP_REDIS_INSTANCE_ADDRESS} --vpc-connector=$OSDU_GCP_VPC_CONNECTOR - include: # pipeline logic - project: "osdu/platform/ci-cd-pipelines" file: "standard-setup.yml" - # build - - local: "devops/osdu/build/seismic-store-service.yml" - - # scan - # fossa - - local: "devops/osdu/scanners/fossa-node.yml" - - # lint - - local: "/devops/osdu/scanners/lint-node.yml" - - # scan for secrets - - local: "/devops/osdu/scanners/scan-for-secrets-node.yml" - - # containerize - - project: "osdu/platform/ci-cd-pipelines" - file: "containerize/seismic-store-service.yml" - - # aws - - project: "osdu/platform/ci-cd-pipelines" - file: "cloud-providers/aws-global.yml" - - - local: "/devops/aws/awstest.yml" - - # deploy - #azure - - local: "/devops/osdu/cloud-providers/azure-seismic-store-service.yml" - - #ibm - - project: "osdu/platform/ci-cd-pipelines" - file: "cloud-providers/ibm-seismic-store-service.yml" - - #gcp - - project: "osdu/platform/ci-cd-pipelines" - ref: "master" - file: "cloud-providers/osdu-gcp-cloudrun.yml" - -osdu-gcp-test: - script: - - echo "Tests not implemented" - extends: - - .osdu-gcp-variables - only: - variables: - - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE != 'python' - -osdu-gcp-test-python: - script: - - echo "Tests not implemented" - only: - variables: - - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE == 'python' - -osdu-gcp-containerize-gitlab: - stage: containerize - needs: ["compile-and-unit-test"] - tags: ["osdu-medium"] - extends: .osdu-gcp-variables - image: docker:19.03 - cache: {} - allow_failure: true - script: - - export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:latest" ; fi - - docker build -t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --file docker/runtime.Dockerfile . - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker push $CI_REGISTRY_IMAGE/osdu-gcp - only: - variables: - - $OSDU_GCP == 'true' + # app + # sdms + - local: "app/sdms/.gitlab-ci.yml" \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/app/sdms/.devcontainer/Dockerfile similarity index 100% rename from .devcontainer/Dockerfile rename to app/sdms/.devcontainer/Dockerfile diff --git a/.devcontainer/README.md b/app/sdms/.devcontainer/README.md similarity index 100% rename from .devcontainer/README.md rename to app/sdms/.devcontainer/README.md diff --git a/.devcontainer/bootstrap.sh b/app/sdms/.devcontainer/bootstrap.sh similarity index 100% rename from .devcontainer/bootstrap.sh rename to app/sdms/.devcontainer/bootstrap.sh diff --git a/.devcontainer/devcontainer.json b/app/sdms/.devcontainer/devcontainer.json similarity index 100% rename from .devcontainer/devcontainer.json rename to app/sdms/.devcontainer/devcontainer.json diff --git a/.devcontainer/readme/bash.png b/app/sdms/.devcontainer/readme/bash.png similarity index 100% rename from .devcontainer/readme/bash.png rename to app/sdms/.devcontainer/readme/bash.png diff --git a/.devcontainer/readme/command-palette.png b/app/sdms/.devcontainer/readme/command-palette.png similarity index 100% rename from .devcontainer/readme/command-palette.png rename to app/sdms/.devcontainer/readme/command-palette.png diff --git a/.devcontainer/readme/container.png b/app/sdms/.devcontainer/readme/container.png similarity index 100% rename from .devcontainer/readme/container.png rename to app/sdms/.devcontainer/readme/container.png diff --git a/.dockerignore b/app/sdms/.dockerignore similarity index 100% rename from .dockerignore rename to app/sdms/.dockerignore diff --git a/.fossa.yml b/app/sdms/.fossa.yml similarity index 100% rename from .fossa.yml rename to app/sdms/.fossa.yml diff --git a/.gitattributes b/app/sdms/.gitattributes similarity index 100% rename from .gitattributes rename to app/sdms/.gitattributes diff --git a/.gitignore b/app/sdms/.gitignore similarity index 100% rename from .gitignore rename to app/sdms/.gitignore diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml new file mode 100644 index 00000000..895b8e70 --- /dev/null +++ b/app/sdms/.gitlab-ci.yml @@ -0,0 +1,48 @@ +variables: + #seismic store service variables + ENVIRONMENT: cloud + ISGITLAB: "true" + PORT: 80 + REPLICA: 1 + UTEST_RUNTIME_IMAGE: seistore-svc-runtime + SDMS_MIN_REPLICAS: 1 + SDMS_MAX_REPLICAS: 5 + + #aws variables + AWS_SERVICE: seismic-store + AWS_BUILD_SUBDIR: src/cloud/providers/aws/build-aws + AWS_TEST_SUBDIR: tests + AWS_ENVIRONMENT: dev + AWS_DEPLOY_TARGET: EKS + AWS_EKS_DEPLOYMENT_NAME: os-seismic-store + AWS_BUILDER_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/builder.Dockerfile + AWS_RUNTIME_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/runtime.Dockerfile + # skipping tests here. Using a local file to run tests + AWS_SKIP_TESTS: "true" + #azure variables + AZURE_SERVICE: seismic-store-service + #gcp variables + GCP_APPLICATION_NAME: os-seismic-store-service + GCP_DEPLOY_FILE: $OSDU_GCP_DEPLOY_FILE + GCP_PROJECT: opendes-evt + GCP_SERVICE: seismic-store-service + GCP_VENDOR: gcp + #ibm variables + IBM_OPENSHIFT_PROJECT: og-cicd + IBM_VENDOR: ibm + + OSDU_GCP_LOG_LEVEL: INFO + OSDU_GCP_SERVICE: seismic-store + OSDU_GCP_VENDOR: gcp + OSDU_GCP_APPLICATION_NAME: seismic-store + OSDU_GCP_APPLICATION: seismic-store + OSDU_GCP_ENTITLEMENT_BASE_URL_PATH: /entitlements/v2 + OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY: data-partition-id + OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE: https://community.osdu-gcp.go3-nrg.projects.epam.com/api + OSDU_GCP_DES_SERVICE_HOST_STORAGE: https://os-storage-attcrcktoa-uc.a.run.app/api + OSDU_GCP_ENV_VARS: CLOUDPROVIDER=${OSDU_GCP_CLOUD_PROVIDER},DES_SERVICE_HOST_PARTITION=${OSDU_GCP_PARTITION_API},ENTITLEMENT_BASE_URL_PATH=${OSDU_GCP_ENTITLEMENT_BASE_URL_PATH},DATA_PARTITION_REST_HEADER_KEY=${OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY},DES_SERVICE_HOST_STORAGE=${OSDU_GCP_DES_SERVICE_HOST_STORAGE},DES_SERVICE_HOST_COMPLIANCE=${OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE},SEISTORE_DES_TARGET_AUDIENCE=${GOOGLE_AUDIENCE},SERVICE_CLOUD_PROJECT=${OSDU_GCP_PROJECT},APP_ENVIRONMENT_IDENTIFIER=${TENANT},IMP_SERVICE_ACCOUNT_SIGNER=${OSDU_GCP_IMP_SERVICE_ACCOUNT_SIGNER},DES_SERVICE_HOST_ENTITLEMENT=${OSDU_GCP_ENTITLEMENTS_V2_BASE_URL},SEISTORE_DES_APPKEY=${OSDU_GCP_SEISTORE_DES_APPKEY},DES_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_DES_REDIS_INSTANCE_ADDRESS},DES_REDIS_INSTANCE_PORT=${OSDU_GCP_DES_REDIS_INSTANCE_PORT},LOCKSMAP_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_LOCKSMAP_REDIS_INSTANCE_ADDRESS} --vpc-connector=$OSDU_GCP_VPC_CONNECTOR + +include: + + # build + - local: "app/sdms/devops/osdu/build/seismic-store-service.yml" diff --git a/.husky/pre-commit b/app/sdms/.husky/pre-commit similarity index 100% rename from .husky/pre-commit rename to app/sdms/.husky/pre-commit diff --git a/.mocharc.json b/app/sdms/.mocharc.json similarity index 100% rename from .mocharc.json rename to app/sdms/.mocharc.json diff --git a/CHANGELOG.md b/app/sdms/CHANGELOG.md similarity index 100% rename from CHANGELOG.md rename to app/sdms/CHANGELOG.md diff --git a/LICENSE b/app/sdms/LICENSE similarity index 100% rename from LICENSE rename to app/sdms/LICENSE diff --git a/NOTICE b/app/sdms/NOTICE similarity index 100% rename from NOTICE rename to app/sdms/NOTICE diff --git a/app/sdms/README.md b/app/sdms/README.md new file mode 100644 index 00000000..c5ef000d --- /dev/null +++ b/app/sdms/README.md @@ -0,0 +1,54 @@ +# SEISMIC STORE + +Seismic Store is a cloud-based solution designed to store and manage datasets of any size in the cloud by enabling a secure way to access them through a scoped authorization mechanism. Seismic Store overcomes the object size limitations imposed by a cloud provider, by managing generic datasets as multi independent objects and, therefore, provides a generic, reliable and a better performed solution to handle data on a cloud storage. + +Saving a dataset on a cloud-based storage, as single entity, may be a problem when it exceeds the maximum allowed object size. Adopting a single object storage approach is also not an optimal solution in terms of performance as a single entity cannot be easily uploaded and downloaded directly in parallel. + +Seismic Store is a cloud-based solution composed by restful micro-services, client APIs and tools designed to implement a multi-object storage approach. The system saves objects that compose a dataset as a hierarchical data structure in a cloud storage and the dataset properties as a metadata entry in a no-relational catalogue. Having the datasets stored as multiple independent objects improve the overall performance, as generic I/O operations, for example read or write objects, can be easily parallelized. + +Seismic Store manages data authorization at service level by protecting access to storage bucket resources. Only service authorized users are enabled to directly access a storage resource. The service implements a mechanism that generates an “impersonation token” by authorizing long running/background production jobs to access data without requiring further user interactions. + +![service architecture diagram](docs/sdms-architecture-diagram-plain.png "Service Architecture Diagram") + +```python +# build the service +npm run build + +# start the service +npm run start + +# run integral/unit tests +npm run test + +# run integral/unit tests and generate the code coverage +npm run code-coverage + +# run the regression/e2e test suite +./tests/e2e/run_e2e_tests.sh \ + --seistore-svc-url="seismic store service url" \ + --seistore-svc-api-key="seismic store service api key" \ + --user-idtoken="user or service agent idtoken" \ + --tenant="seistore working tenant" \ + --subproject="seistore working subproject" \ + --admin-email="admin email" + +# run the parallel regression/e2e test suite (add the --run-parallel option) +./tests/e2e/run_e2e_tests.sh \ + --seistore-svc-url="seismic store service url" \ + --seistore-svc-api-key="seismic store service api key" \ + --user-idtoken="user or service agent idtoken" \ + --tenant="seistore working tenant" \ + --subproject="seistore working subproject" \ + --admin-email="admin email" \ + --run-parallel + +# run the linter on sources +tslint -c tslint.json -p tsconfig.json 'src/**/*.ts' +``` + +## Environment configuration + +Environment variables can be provided with a `.env` file in the root of the project to be consumed +by [dotenv](https://github.com/motdotla/dotenv). Environment variables are [preloaded](https://github.com/motdotla/dotenv#preload) +by the `npm start` command with the argument `-r dotenv/config`. A template `.env` file can be found +in `/docs/templates/.env-sample`. diff --git a/devops/aws/awstest.yml b/app/sdms/devops/aws/awstest.yml similarity index 100% rename from devops/aws/awstest.yml rename to app/sdms/devops/aws/awstest.yml diff --git a/devops/aws/requirements.txt b/app/sdms/devops/aws/requirements.txt similarity index 100% rename from devops/aws/requirements.txt rename to app/sdms/devops/aws/requirements.txt diff --git a/devops/azure/chart/Chart.yaml b/app/sdms/devops/azure/chart/Chart.yaml similarity index 100% rename from devops/azure/chart/Chart.yaml rename to app/sdms/devops/azure/chart/Chart.yaml diff --git a/devops/azure/chart/helm-config-rollback.yaml b/app/sdms/devops/azure/chart/helm-config-rollback.yaml similarity index 100% rename from devops/azure/chart/helm-config-rollback.yaml rename to app/sdms/devops/azure/chart/helm-config-rollback.yaml diff --git a/devops/azure/chart/helm-config.yaml b/app/sdms/devops/azure/chart/helm-config.yaml similarity index 100% rename from devops/azure/chart/helm-config.yaml rename to app/sdms/devops/azure/chart/helm-config.yaml diff --git a/devops/azure/chart/templates/azure-istio-auth-policy.yaml b/app/sdms/devops/azure/chart/templates/azure-istio-auth-policy.yaml similarity index 100% rename from devops/azure/chart/templates/azure-istio-auth-policy.yaml rename to app/sdms/devops/azure/chart/templates/azure-istio-auth-policy.yaml diff --git a/devops/azure/chart/templates/configmap.yaml b/app/sdms/devops/azure/chart/templates/configmap.yaml similarity index 100% rename from devops/azure/chart/templates/configmap.yaml rename to app/sdms/devops/azure/chart/templates/configmap.yaml diff --git a/devops/azure/chart/templates/deployment.yaml b/app/sdms/devops/azure/chart/templates/deployment.yaml similarity index 100% rename from devops/azure/chart/templates/deployment.yaml rename to app/sdms/devops/azure/chart/templates/deployment.yaml diff --git a/devops/azure/chart/templates/hpa.yaml b/app/sdms/devops/azure/chart/templates/hpa.yaml similarity index 100% rename from devops/azure/chart/templates/hpa.yaml rename to app/sdms/devops/azure/chart/templates/hpa.yaml diff --git a/devops/azure/chart/templates/service.yaml b/app/sdms/devops/azure/chart/templates/service.yaml similarity index 100% rename from devops/azure/chart/templates/service.yaml rename to app/sdms/devops/azure/chart/templates/service.yaml diff --git a/devops/azure/development-pipeline.yml b/app/sdms/devops/azure/development-pipeline.yml similarity index 100% rename from devops/azure/development-pipeline.yml rename to app/sdms/devops/azure/development-pipeline.yml diff --git a/devops/azure/pipeline.yml b/app/sdms/devops/azure/pipeline.yml similarity index 100% rename from devops/azure/pipeline.yml rename to app/sdms/devops/azure/pipeline.yml diff --git a/devops/azure/template/build-stage.yml b/app/sdms/devops/azure/template/build-stage.yml similarity index 100% rename from devops/azure/template/build-stage.yml rename to app/sdms/devops/azure/template/build-stage.yml diff --git a/devops/azure/template/deploy-stage.yml b/app/sdms/devops/azure/template/deploy-stage.yml similarity index 100% rename from devops/azure/template/deploy-stage.yml rename to app/sdms/devops/azure/template/deploy-stage.yml diff --git a/devops/azure/template/task/aks-deployment-steps.yml b/app/sdms/devops/azure/template/task/aks-deployment-steps.yml similarity index 100% rename from devops/azure/template/task/aks-deployment-steps.yml rename to app/sdms/devops/azure/template/task/aks-deployment-steps.yml diff --git a/devops/azure/template/task/build-and-unit-test.yml b/app/sdms/devops/azure/template/task/build-and-unit-test.yml similarity index 100% rename from devops/azure/template/task/build-and-unit-test.yml rename to app/sdms/devops/azure/template/task/build-and-unit-test.yml diff --git a/devops/azure/template/task/build-runtime-image.yml b/app/sdms/devops/azure/template/task/build-runtime-image.yml similarity index 100% rename from devops/azure/template/task/build-runtime-image.yml rename to app/sdms/devops/azure/template/task/build-runtime-image.yml diff --git a/devops/azure/template/task/build-steps.yml b/app/sdms/devops/azure/template/task/build-steps.yml similarity index 100% rename from devops/azure/template/task/build-steps.yml rename to app/sdms/devops/azure/template/task/build-steps.yml diff --git a/devops/azure/template/task/copy-code-coverage-report.yml b/app/sdms/devops/azure/template/task/copy-code-coverage-report.yml similarity index 100% rename from devops/azure/template/task/copy-code-coverage-report.yml rename to app/sdms/devops/azure/template/task/copy-code-coverage-report.yml diff --git a/devops/azure/template/task/deployment-helm-steps.yml b/app/sdms/devops/azure/template/task/deployment-helm-steps.yml similarity index 100% rename from devops/azure/template/task/deployment-helm-steps.yml rename to app/sdms/devops/azure/template/task/deployment-helm-steps.yml diff --git a/devops/azure/template/task/deployment-image-steps.yml b/app/sdms/devops/azure/template/task/deployment-image-steps.yml similarity index 100% rename from devops/azure/template/task/deployment-image-steps.yml rename to app/sdms/devops/azure/template/task/deployment-image-steps.yml diff --git a/devops/azure/template/task/download-build-artifact.yml b/app/sdms/devops/azure/template/task/download-build-artifact.yml similarity index 100% rename from devops/azure/template/task/download-build-artifact.yml rename to app/sdms/devops/azure/template/task/download-build-artifact.yml diff --git a/devops/azure/template/task/e2e-steps.yml b/app/sdms/devops/azure/template/task/e2e-steps.yml similarity index 100% rename from devops/azure/template/task/e2e-steps.yml rename to app/sdms/devops/azure/template/task/e2e-steps.yml diff --git a/devops/azure/template/task/e2e.yml b/app/sdms/devops/azure/template/task/e2e.yml similarity index 100% rename from devops/azure/template/task/e2e.yml rename to app/sdms/devops/azure/template/task/e2e.yml diff --git a/devops/azure/template/task/flux-service-wait.yml b/app/sdms/devops/azure/template/task/flux-service-wait.yml similarity index 100% rename from devops/azure/template/task/flux-service-wait.yml rename to app/sdms/devops/azure/template/task/flux-service-wait.yml diff --git a/devops/azure/template/task/generate-token.yml b/app/sdms/devops/azure/template/task/generate-token.yml similarity index 100% rename from devops/azure/template/task/generate-token.yml rename to app/sdms/devops/azure/template/task/generate-token.yml diff --git a/devops/azure/template/task/gitops.yml b/app/sdms/devops/azure/template/task/gitops.yml similarity index 100% rename from devops/azure/template/task/gitops.yml rename to app/sdms/devops/azure/template/task/gitops.yml diff --git a/devops/azure/template/task/helm-template.yml b/app/sdms/devops/azure/template/task/helm-template.yml similarity index 100% rename from devops/azure/template/task/helm-template.yml rename to app/sdms/devops/azure/template/task/helm-template.yml diff --git a/devops/azure/template/task/install-e2e-requirements.yml b/app/sdms/devops/azure/template/task/install-e2e-requirements.yml similarity index 100% rename from devops/azure/template/task/install-e2e-requirements.yml rename to app/sdms/devops/azure/template/task/install-e2e-requirements.yml diff --git a/devops/azure/template/task/kube.yml b/app/sdms/devops/azure/template/task/kube.yml similarity index 100% rename from devops/azure/template/task/kube.yml rename to app/sdms/devops/azure/template/task/kube.yml diff --git a/devops/azure/template/task/publish-build-artifact.yml b/app/sdms/devops/azure/template/task/publish-build-artifact.yml similarity index 100% rename from devops/azure/template/task/publish-build-artifact.yml rename to app/sdms/devops/azure/template/task/publish-build-artifact.yml diff --git a/devops/azure/template/task/publish-code-coverage.yml b/app/sdms/devops/azure/template/task/publish-code-coverage.yml similarity index 100% rename from devops/azure/template/task/publish-code-coverage.yml rename to app/sdms/devops/azure/template/task/publish-code-coverage.yml diff --git a/devops/azure/template/task/publish-unit-test-results.yml b/app/sdms/devops/azure/template/task/publish-unit-test-results.yml similarity index 100% rename from devops/azure/template/task/publish-unit-test-results.yml rename to app/sdms/devops/azure/template/task/publish-unit-test-results.yml diff --git a/devops/azure/template/task/push-runtime-image.yml b/app/sdms/devops/azure/template/task/push-runtime-image.yml similarity index 100% rename from devops/azure/template/task/push-runtime-image.yml rename to app/sdms/devops/azure/template/task/push-runtime-image.yml diff --git a/devops/azure/template/task/rollback-steps.yml b/app/sdms/devops/azure/template/task/rollback-steps.yml similarity index 100% rename from devops/azure/template/task/rollback-steps.yml rename to app/sdms/devops/azure/template/task/rollback-steps.yml diff --git a/devops/azure/template/task/rollback.yml b/app/sdms/devops/azure/template/task/rollback.yml similarity index 100% rename from devops/azure/template/task/rollback.yml rename to app/sdms/devops/azure/template/task/rollback.yml diff --git a/devops/azure/template/task/run-e2e.yml b/app/sdms/devops/azure/template/task/run-e2e.yml similarity index 100% rename from devops/azure/template/task/run-e2e.yml rename to app/sdms/devops/azure/template/task/run-e2e.yml diff --git a/devops/docker/detect_secrets/.secrets.baseline b/app/sdms/devops/docker/detect_secrets/.secrets.baseline similarity index 100% rename from devops/docker/detect_secrets/.secrets.baseline rename to app/sdms/devops/docker/detect_secrets/.secrets.baseline diff --git a/devops/docker/detect_secrets/Dockerfile b/app/sdms/devops/docker/detect_secrets/Dockerfile similarity index 100% rename from devops/docker/detect_secrets/Dockerfile rename to app/sdms/devops/docker/detect_secrets/Dockerfile diff --git a/devops/docker/detect_secrets/README.md b/app/sdms/devops/docker/detect_secrets/README.md similarity index 100% rename from devops/docker/detect_secrets/README.md rename to app/sdms/devops/docker/detect_secrets/README.md diff --git a/devops/osdu/build/seismic-store-service.yml b/app/sdms/devops/osdu/build/seismic-store-service.yml similarity index 92% rename from devops/osdu/build/seismic-store-service.yml rename to app/sdms/devops/osdu/build/seismic-store-service.yml index 9f0ca562..a7a8de35 100644 --- a/devops/osdu/build/seismic-store-service.yml +++ b/app/sdms/devops/osdu/build/seismic-store-service.yml @@ -12,3 +12,6 @@ compile-and-unit-test: script: - chmod +x devops/scripts/utest_build_run.sh - devops/scripts/utest_build_run.sh + only: + changes: + - app/sdms diff --git a/devops/osdu/cloud-providers/azure-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml similarity index 100% rename from devops/osdu/cloud-providers/azure-seismic-store-service.yml rename to app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml diff --git a/devops/osdu/scanners/fossa-node.yml b/app/sdms/devops/osdu/scanners/fossa-node.yml similarity index 100% rename from devops/osdu/scanners/fossa-node.yml rename to app/sdms/devops/osdu/scanners/fossa-node.yml diff --git a/devops/osdu/scanners/lint-node.yml b/app/sdms/devops/osdu/scanners/lint-node.yml similarity index 100% rename from devops/osdu/scanners/lint-node.yml rename to app/sdms/devops/osdu/scanners/lint-node.yml diff --git a/devops/osdu/scanners/scan-for-secrets-node.yml b/app/sdms/devops/osdu/scanners/scan-for-secrets-node.yml similarity index 100% rename from devops/osdu/scanners/scan-for-secrets-node.yml rename to app/sdms/devops/osdu/scanners/scan-for-secrets-node.yml diff --git a/devops/scripts/aws_jwt_client.py b/app/sdms/devops/scripts/aws_jwt_client.py similarity index 100% rename from devops/scripts/aws_jwt_client.py rename to app/sdms/devops/scripts/aws_jwt_client.py diff --git a/devops/scripts/azure_jwt_client.py b/app/sdms/devops/scripts/azure_jwt_client.py similarity index 100% rename from devops/scripts/azure_jwt_client.py rename to app/sdms/devops/scripts/azure_jwt_client.py diff --git a/devops/scripts/cssfix.js b/app/sdms/devops/scripts/cssfix.js similarity index 100% rename from devops/scripts/cssfix.js rename to app/sdms/devops/scripts/cssfix.js diff --git a/devops/scripts/genchangelog.sh b/app/sdms/devops/scripts/genchangelog.sh similarity index 100% rename from devops/scripts/genchangelog.sh rename to app/sdms/devops/scripts/genchangelog.sh diff --git a/devops/scripts/ibm_jwt_client.py b/app/sdms/devops/scripts/ibm_jwt_client.py similarity index 100% rename from devops/scripts/ibm_jwt_client.py rename to app/sdms/devops/scripts/ibm_jwt_client.py diff --git a/devops/scripts/utest_build_run.sh b/app/sdms/devops/scripts/utest_build_run.sh similarity index 100% rename from devops/scripts/utest_build_run.sh rename to app/sdms/devops/scripts/utest_build_run.sh diff --git a/docker/README.md b/app/sdms/docker/README.md similarity index 100% rename from docker/README.md rename to app/sdms/docker/README.md diff --git a/docker/builder.Dockerfile b/app/sdms/docker/builder.Dockerfile similarity index 100% rename from docker/builder.Dockerfile rename to app/sdms/docker/builder.Dockerfile diff --git a/docker/runtime.Dockerfile b/app/sdms/docker/runtime.Dockerfile similarity index 100% rename from docker/runtime.Dockerfile rename to app/sdms/docker/runtime.Dockerfile diff --git a/docs/api/openapi.osdu.yaml b/app/sdms/docs/api/openapi.osdu.yaml similarity index 100% rename from docs/api/openapi.osdu.yaml rename to app/sdms/docs/api/openapi.osdu.yaml diff --git a/docs/api/openapi.yaml b/app/sdms/docs/api/openapi.yaml similarity index 100% rename from docs/api/openapi.yaml rename to app/sdms/docs/api/openapi.yaml diff --git a/docs/sdms-architecture-diagram-plain.png b/app/sdms/docs/sdms-architecture-diagram-plain.png similarity index 100% rename from docs/sdms-architecture-diagram-plain.png rename to app/sdms/docs/sdms-architecture-diagram-plain.png diff --git a/docs/sdms-architecture-diagram.png b/app/sdms/docs/sdms-architecture-diagram.png similarity index 100% rename from docs/sdms-architecture-diagram.png rename to app/sdms/docs/sdms-architecture-diagram.png diff --git a/docs/templates/.env-sample-aws b/app/sdms/docs/templates/.env-sample-aws similarity index 100% rename from docs/templates/.env-sample-aws rename to app/sdms/docs/templates/.env-sample-aws diff --git a/docs/templates/.env-sample-azure b/app/sdms/docs/templates/.env-sample-azure similarity index 100% rename from docs/templates/.env-sample-azure rename to app/sdms/docs/templates/.env-sample-azure diff --git a/docs/templates/.env-sample-google b/app/sdms/docs/templates/.env-sample-google similarity index 100% rename from docs/templates/.env-sample-google rename to app/sdms/docs/templates/.env-sample-google diff --git a/npm-shrinkwrap.json b/app/sdms/npm-shrinkwrap.json similarity index 100% rename from npm-shrinkwrap.json rename to app/sdms/npm-shrinkwrap.json diff --git a/package.json b/app/sdms/package.json similarity index 100% rename from package.json rename to app/sdms/package.json diff --git a/provider/seismic-store-gcp/cloudbuild/cloudbuild.yaml b/app/sdms/provider/seismic-store-gcp/cloudbuild/cloudbuild.yaml similarity index 100% rename from provider/seismic-store-gcp/cloudbuild/cloudbuild.yaml rename to app/sdms/provider/seismic-store-gcp/cloudbuild/cloudbuild.yaml diff --git a/src/auth/auth.ts b/app/sdms/src/auth/auth.ts similarity index 100% rename from src/auth/auth.ts rename to app/sdms/src/auth/auth.ts diff --git a/src/auth/groups.ts b/app/sdms/src/auth/groups.ts similarity index 100% rename from src/auth/groups.ts rename to app/sdms/src/auth/groups.ts diff --git a/src/auth/index.ts b/app/sdms/src/auth/index.ts similarity index 100% rename from src/auth/index.ts rename to app/sdms/src/auth/index.ts diff --git a/src/auth/providers/generic/auth.ts b/app/sdms/src/auth/providers/generic/auth.ts similarity index 100% rename from src/auth/providers/generic/auth.ts rename to app/sdms/src/auth/providers/generic/auth.ts diff --git a/src/auth/providers/generic/index.ts b/app/sdms/src/auth/providers/generic/index.ts similarity index 100% rename from src/auth/providers/generic/index.ts rename to app/sdms/src/auth/providers/generic/index.ts diff --git a/src/auth/providers/index.ts b/app/sdms/src/auth/providers/index.ts similarity index 100% rename from src/auth/providers/index.ts rename to app/sdms/src/auth/providers/index.ts diff --git a/src/auth/roles.ts b/app/sdms/src/auth/roles.ts similarity index 100% rename from src/auth/roles.ts rename to app/sdms/src/auth/roles.ts diff --git a/src/cloud/cloud.ts b/app/sdms/src/cloud/cloud.ts similarity index 100% rename from src/cloud/cloud.ts rename to app/sdms/src/cloud/cloud.ts diff --git a/src/cloud/config.ts b/app/sdms/src/cloud/config.ts similarity index 100% rename from src/cloud/config.ts rename to app/sdms/src/cloud/config.ts diff --git a/src/cloud/credentials.ts b/app/sdms/src/cloud/credentials.ts similarity index 100% rename from src/cloud/credentials.ts rename to app/sdms/src/cloud/credentials.ts diff --git a/src/cloud/dataecosystem.ts b/app/sdms/src/cloud/dataecosystem.ts similarity index 100% rename from src/cloud/dataecosystem.ts rename to app/sdms/src/cloud/dataecosystem.ts diff --git a/src/cloud/index.ts b/app/sdms/src/cloud/index.ts similarity index 100% rename from src/cloud/index.ts rename to app/sdms/src/cloud/index.ts diff --git a/src/cloud/journal.ts b/app/sdms/src/cloud/journal.ts similarity index 100% rename from src/cloud/journal.ts rename to app/sdms/src/cloud/journal.ts diff --git a/src/cloud/logger.ts b/app/sdms/src/cloud/logger.ts similarity index 100% rename from src/cloud/logger.ts rename to app/sdms/src/cloud/logger.ts diff --git a/src/cloud/providers/aws/build-aws/build-info.py b/app/sdms/src/cloud/providers/aws/build-aws/build-info.py similarity index 100% rename from src/cloud/providers/aws/build-aws/build-info.py rename to app/sdms/src/cloud/providers/aws/build-aws/build-info.py diff --git a/src/cloud/providers/aws/build-aws/builder.Dockerfile b/app/sdms/src/cloud/providers/aws/build-aws/builder.Dockerfile similarity index 100% rename from src/cloud/providers/aws/build-aws/builder.Dockerfile rename to app/sdms/src/cloud/providers/aws/build-aws/builder.Dockerfile diff --git a/src/cloud/providers/aws/build-aws/buildspec.yaml b/app/sdms/src/cloud/providers/aws/build-aws/buildspec.yaml similarity index 100% rename from src/cloud/providers/aws/build-aws/buildspec.yaml rename to app/sdms/src/cloud/providers/aws/build-aws/buildspec.yaml diff --git a/src/cloud/providers/aws/build-aws/entrypoint.sh b/app/sdms/src/cloud/providers/aws/build-aws/entrypoint.sh old mode 100755 new mode 100644 similarity index 100% rename from src/cloud/providers/aws/build-aws/entrypoint.sh rename to app/sdms/src/cloud/providers/aws/build-aws/entrypoint.sh diff --git a/src/cloud/providers/aws/build-aws/runtime.Dockerfile b/app/sdms/src/cloud/providers/aws/build-aws/runtime.Dockerfile similarity index 100% rename from src/cloud/providers/aws/build-aws/runtime.Dockerfile rename to app/sdms/src/cloud/providers/aws/build-aws/runtime.Dockerfile diff --git a/src/cloud/providers/aws/build-aws/ssl.sh b/app/sdms/src/cloud/providers/aws/build-aws/ssl.sh old mode 100755 new mode 100644 similarity index 100% rename from src/cloud/providers/aws/build-aws/ssl.sh rename to app/sdms/src/cloud/providers/aws/build-aws/ssl.sh diff --git a/src/cloud/providers/aws/config.ts b/app/sdms/src/cloud/providers/aws/config.ts similarity index 100% rename from src/cloud/providers/aws/config.ts rename to app/sdms/src/cloud/providers/aws/config.ts diff --git a/src/cloud/providers/aws/credentials.ts b/app/sdms/src/cloud/providers/aws/credentials.ts similarity index 100% rename from src/cloud/providers/aws/credentials.ts rename to app/sdms/src/cloud/providers/aws/credentials.ts diff --git a/src/cloud/providers/aws/dataecosystem.ts b/app/sdms/src/cloud/providers/aws/dataecosystem.ts similarity index 100% rename from src/cloud/providers/aws/dataecosystem.ts rename to app/sdms/src/cloud/providers/aws/dataecosystem.ts diff --git a/src/cloud/providers/aws/dynamodb.ts b/app/sdms/src/cloud/providers/aws/dynamodb.ts similarity index 100% rename from src/cloud/providers/aws/dynamodb.ts rename to app/sdms/src/cloud/providers/aws/dynamodb.ts diff --git a/src/cloud/providers/aws/index.ts b/app/sdms/src/cloud/providers/aws/index.ts similarity index 100% rename from src/cloud/providers/aws/index.ts rename to app/sdms/src/cloud/providers/aws/index.ts diff --git a/src/cloud/providers/aws/logger.ts b/app/sdms/src/cloud/providers/aws/logger.ts similarity index 100% rename from src/cloud/providers/aws/logger.ts rename to app/sdms/src/cloud/providers/aws/logger.ts diff --git a/src/cloud/providers/aws/seistore.ts b/app/sdms/src/cloud/providers/aws/seistore.ts similarity index 100% rename from src/cloud/providers/aws/seistore.ts rename to app/sdms/src/cloud/providers/aws/seistore.ts diff --git a/src/cloud/providers/aws/ssmhelper.ts b/app/sdms/src/cloud/providers/aws/ssmhelper.ts similarity index 100% rename from src/cloud/providers/aws/ssmhelper.ts rename to app/sdms/src/cloud/providers/aws/ssmhelper.ts diff --git a/src/cloud/providers/aws/storage.ts b/app/sdms/src/cloud/providers/aws/storage.ts similarity index 100% rename from src/cloud/providers/aws/storage.ts rename to app/sdms/src/cloud/providers/aws/storage.ts diff --git a/src/cloud/providers/aws/stshelper.ts b/app/sdms/src/cloud/providers/aws/stshelper.ts similarity index 100% rename from src/cloud/providers/aws/stshelper.ts rename to app/sdms/src/cloud/providers/aws/stshelper.ts diff --git a/src/cloud/providers/aws/trace.ts b/app/sdms/src/cloud/providers/aws/trace.ts similarity index 100% rename from src/cloud/providers/aws/trace.ts rename to app/sdms/src/cloud/providers/aws/trace.ts diff --git a/src/cloud/providers/azure/cloudstorage.ts b/app/sdms/src/cloud/providers/azure/cloudstorage.ts similarity index 100% rename from src/cloud/providers/azure/cloudstorage.ts rename to app/sdms/src/cloud/providers/azure/cloudstorage.ts diff --git a/src/cloud/providers/azure/config.ts b/app/sdms/src/cloud/providers/azure/config.ts similarity index 100% rename from src/cloud/providers/azure/config.ts rename to app/sdms/src/cloud/providers/azure/config.ts diff --git a/src/cloud/providers/azure/cosmosdb.ts b/app/sdms/src/cloud/providers/azure/cosmosdb.ts similarity index 100% rename from src/cloud/providers/azure/cosmosdb.ts rename to app/sdms/src/cloud/providers/azure/cosmosdb.ts diff --git a/src/cloud/providers/azure/credentials.ts b/app/sdms/src/cloud/providers/azure/credentials.ts similarity index 100% rename from src/cloud/providers/azure/credentials.ts rename to app/sdms/src/cloud/providers/azure/credentials.ts diff --git a/src/cloud/providers/azure/dataecosystem.ts b/app/sdms/src/cloud/providers/azure/dataecosystem.ts similarity index 100% rename from src/cloud/providers/azure/dataecosystem.ts rename to app/sdms/src/cloud/providers/azure/dataecosystem.ts diff --git a/src/cloud/providers/azure/index.ts b/app/sdms/src/cloud/providers/azure/index.ts similarity index 100% rename from src/cloud/providers/azure/index.ts rename to app/sdms/src/cloud/providers/azure/index.ts diff --git a/src/cloud/providers/azure/insights.ts b/app/sdms/src/cloud/providers/azure/insights.ts similarity index 100% rename from src/cloud/providers/azure/insights.ts rename to app/sdms/src/cloud/providers/azure/insights.ts diff --git a/src/cloud/providers/azure/keyvault.ts b/app/sdms/src/cloud/providers/azure/keyvault.ts similarity index 100% rename from src/cloud/providers/azure/keyvault.ts rename to app/sdms/src/cloud/providers/azure/keyvault.ts diff --git a/src/cloud/providers/azure/seistore.ts b/app/sdms/src/cloud/providers/azure/seistore.ts similarity index 100% rename from src/cloud/providers/azure/seistore.ts rename to app/sdms/src/cloud/providers/azure/seistore.ts diff --git a/src/cloud/providers/azure/trace.ts b/app/sdms/src/cloud/providers/azure/trace.ts similarity index 100% rename from src/cloud/providers/azure/trace.ts rename to app/sdms/src/cloud/providers/azure/trace.ts diff --git a/src/cloud/providers/google/config.ts b/app/sdms/src/cloud/providers/google/config.ts similarity index 100% rename from src/cloud/providers/google/config.ts rename to app/sdms/src/cloud/providers/google/config.ts diff --git a/src/cloud/providers/google/credentials.ts b/app/sdms/src/cloud/providers/google/credentials.ts similarity index 100% rename from src/cloud/providers/google/credentials.ts rename to app/sdms/src/cloud/providers/google/credentials.ts diff --git a/src/cloud/providers/google/dataecosystem.ts b/app/sdms/src/cloud/providers/google/dataecosystem.ts similarity index 100% rename from src/cloud/providers/google/dataecosystem.ts rename to app/sdms/src/cloud/providers/google/dataecosystem.ts diff --git a/src/cloud/providers/google/datastore.ts b/app/sdms/src/cloud/providers/google/datastore.ts similarity index 100% rename from src/cloud/providers/google/datastore.ts rename to app/sdms/src/cloud/providers/google/datastore.ts diff --git a/src/cloud/providers/google/gcs.ts b/app/sdms/src/cloud/providers/google/gcs.ts similarity index 100% rename from src/cloud/providers/google/gcs.ts rename to app/sdms/src/cloud/providers/google/gcs.ts diff --git a/src/cloud/providers/google/index.ts b/app/sdms/src/cloud/providers/google/index.ts similarity index 100% rename from src/cloud/providers/google/index.ts rename to app/sdms/src/cloud/providers/google/index.ts diff --git a/src/cloud/providers/google/logger.ts b/app/sdms/src/cloud/providers/google/logger.ts similarity index 100% rename from src/cloud/providers/google/logger.ts rename to app/sdms/src/cloud/providers/google/logger.ts diff --git a/src/cloud/providers/google/secrets.ts b/app/sdms/src/cloud/providers/google/secrets.ts similarity index 100% rename from src/cloud/providers/google/secrets.ts rename to app/sdms/src/cloud/providers/google/secrets.ts diff --git a/src/cloud/providers/google/seistore.ts b/app/sdms/src/cloud/providers/google/seistore.ts similarity index 100% rename from src/cloud/providers/google/seistore.ts rename to app/sdms/src/cloud/providers/google/seistore.ts diff --git a/src/cloud/providers/google/trace.ts b/app/sdms/src/cloud/providers/google/trace.ts similarity index 100% rename from src/cloud/providers/google/trace.ts rename to app/sdms/src/cloud/providers/google/trace.ts diff --git a/src/cloud/providers/ibm/config.ts b/app/sdms/src/cloud/providers/ibm/config.ts similarity index 100% rename from src/cloud/providers/ibm/config.ts rename to app/sdms/src/cloud/providers/ibm/config.ts diff --git a/src/cloud/providers/ibm/cos.ts b/app/sdms/src/cloud/providers/ibm/cos.ts similarity index 100% rename from src/cloud/providers/ibm/cos.ts rename to app/sdms/src/cloud/providers/ibm/cos.ts diff --git a/src/cloud/providers/ibm/credentials.ts b/app/sdms/src/cloud/providers/ibm/credentials.ts similarity index 100% rename from src/cloud/providers/ibm/credentials.ts rename to app/sdms/src/cloud/providers/ibm/credentials.ts diff --git a/src/cloud/providers/ibm/dataecosystem.ts b/app/sdms/src/cloud/providers/ibm/dataecosystem.ts similarity index 100% rename from src/cloud/providers/ibm/dataecosystem.ts rename to app/sdms/src/cloud/providers/ibm/dataecosystem.ts diff --git a/src/cloud/providers/ibm/datastore.ts b/app/sdms/src/cloud/providers/ibm/datastore.ts similarity index 100% rename from src/cloud/providers/ibm/datastore.ts rename to app/sdms/src/cloud/providers/ibm/datastore.ts diff --git a/src/cloud/providers/ibm/index.ts b/app/sdms/src/cloud/providers/ibm/index.ts similarity index 100% rename from src/cloud/providers/ibm/index.ts rename to app/sdms/src/cloud/providers/ibm/index.ts diff --git a/src/cloud/providers/ibm/logger.ts b/app/sdms/src/cloud/providers/ibm/logger.ts similarity index 100% rename from src/cloud/providers/ibm/logger.ts rename to app/sdms/src/cloud/providers/ibm/logger.ts diff --git a/src/cloud/providers/ibm/seistore.ts b/app/sdms/src/cloud/providers/ibm/seistore.ts similarity index 100% rename from src/cloud/providers/ibm/seistore.ts rename to app/sdms/src/cloud/providers/ibm/seistore.ts diff --git a/src/cloud/providers/ibm/stshelper.ts b/app/sdms/src/cloud/providers/ibm/stshelper.ts similarity index 100% rename from src/cloud/providers/ibm/stshelper.ts rename to app/sdms/src/cloud/providers/ibm/stshelper.ts diff --git a/src/cloud/providers/ibm/trace.ts b/app/sdms/src/cloud/providers/ibm/trace.ts similarity index 100% rename from src/cloud/providers/ibm/trace.ts rename to app/sdms/src/cloud/providers/ibm/trace.ts diff --git a/src/cloud/providers/index.ts b/app/sdms/src/cloud/providers/index.ts similarity index 100% rename from src/cloud/providers/index.ts rename to app/sdms/src/cloud/providers/index.ts diff --git a/src/cloud/seistore.ts b/app/sdms/src/cloud/seistore.ts similarity index 100% rename from src/cloud/seistore.ts rename to app/sdms/src/cloud/seistore.ts diff --git a/src/cloud/shared/queue.ts b/app/sdms/src/cloud/shared/queue.ts similarity index 100% rename from src/cloud/shared/queue.ts rename to app/sdms/src/cloud/shared/queue.ts diff --git a/src/cloud/storage.ts b/app/sdms/src/cloud/storage.ts similarity index 100% rename from src/cloud/storage.ts rename to app/sdms/src/cloud/storage.ts diff --git a/src/cloud/trace.ts b/app/sdms/src/cloud/trace.ts similarity index 100% rename from src/cloud/trace.ts rename to app/sdms/src/cloud/trace.ts diff --git a/src/dataecosystem/compliance.ts b/app/sdms/src/dataecosystem/compliance.ts similarity index 100% rename from src/dataecosystem/compliance.ts rename to app/sdms/src/dataecosystem/compliance.ts diff --git a/src/dataecosystem/entitlement.ts b/app/sdms/src/dataecosystem/entitlement.ts similarity index 100% rename from src/dataecosystem/entitlement.ts rename to app/sdms/src/dataecosystem/entitlement.ts diff --git a/src/dataecosystem/index.ts b/app/sdms/src/dataecosystem/index.ts similarity index 100% rename from src/dataecosystem/index.ts rename to app/sdms/src/dataecosystem/index.ts diff --git a/src/dataecosystem/storage.ts b/app/sdms/src/dataecosystem/storage.ts similarity index 100% rename from src/dataecosystem/storage.ts rename to app/sdms/src/dataecosystem/storage.ts diff --git a/src/dataecosystem/utils.ts b/app/sdms/src/dataecosystem/utils.ts similarity index 100% rename from src/dataecosystem/utils.ts rename to app/sdms/src/dataecosystem/utils.ts diff --git a/src/metrics/index.ts b/app/sdms/src/metrics/index.ts similarity index 100% rename from src/metrics/index.ts rename to app/sdms/src/metrics/index.ts diff --git a/src/metrics/metrics.ts b/app/sdms/src/metrics/metrics.ts similarity index 100% rename from src/metrics/metrics.ts rename to app/sdms/src/metrics/metrics.ts diff --git a/src/server/index.ts b/app/sdms/src/server/index.ts similarity index 100% rename from src/server/index.ts rename to app/sdms/src/server/index.ts diff --git a/src/server/server-start.ts b/app/sdms/src/server/server-start.ts similarity index 100% rename from src/server/server-start.ts rename to app/sdms/src/server/server-start.ts diff --git a/src/server/server.ts b/app/sdms/src/server/server.ts similarity index 100% rename from src/server/server.ts rename to app/sdms/src/server/server.ts diff --git a/src/services/dataset/dao.ts b/app/sdms/src/services/dataset/dao.ts similarity index 100% rename from src/services/dataset/dao.ts rename to app/sdms/src/services/dataset/dao.ts diff --git a/src/services/dataset/handler.ts b/app/sdms/src/services/dataset/handler.ts similarity index 100% rename from src/services/dataset/handler.ts rename to app/sdms/src/services/dataset/handler.ts diff --git a/src/services/dataset/index.ts b/app/sdms/src/services/dataset/index.ts similarity index 100% rename from src/services/dataset/index.ts rename to app/sdms/src/services/dataset/index.ts diff --git a/src/services/dataset/locker.ts b/app/sdms/src/services/dataset/locker.ts similarity index 100% rename from src/services/dataset/locker.ts rename to app/sdms/src/services/dataset/locker.ts diff --git a/src/services/dataset/model.ts b/app/sdms/src/services/dataset/model.ts similarity index 100% rename from src/services/dataset/model.ts rename to app/sdms/src/services/dataset/model.ts diff --git a/src/services/dataset/optype.ts b/app/sdms/src/services/dataset/optype.ts similarity index 100% rename from src/services/dataset/optype.ts rename to app/sdms/src/services/dataset/optype.ts diff --git a/src/services/dataset/parser.ts b/app/sdms/src/services/dataset/parser.ts similarity index 100% rename from src/services/dataset/parser.ts rename to app/sdms/src/services/dataset/parser.ts diff --git a/src/services/dataset/service.ts b/app/sdms/src/services/dataset/service.ts similarity index 100% rename from src/services/dataset/service.ts rename to app/sdms/src/services/dataset/service.ts diff --git a/src/services/general/handler.ts b/app/sdms/src/services/general/handler.ts similarity index 100% rename from src/services/general/handler.ts rename to app/sdms/src/services/general/handler.ts diff --git a/src/services/general/optype.ts b/app/sdms/src/services/general/optype.ts similarity index 100% rename from src/services/general/optype.ts rename to app/sdms/src/services/general/optype.ts diff --git a/src/services/general/service.ts b/app/sdms/src/services/general/service.ts similarity index 100% rename from src/services/general/service.ts rename to app/sdms/src/services/general/service.ts diff --git a/src/services/impersonation_token/handler.ts b/app/sdms/src/services/impersonation_token/handler.ts similarity index 100% rename from src/services/impersonation_token/handler.ts rename to app/sdms/src/services/impersonation_token/handler.ts diff --git a/src/services/impersonation_token/model.ts b/app/sdms/src/services/impersonation_token/model.ts similarity index 100% rename from src/services/impersonation_token/model.ts rename to app/sdms/src/services/impersonation_token/model.ts diff --git a/src/services/impersonation_token/optype.ts b/app/sdms/src/services/impersonation_token/optype.ts similarity index 100% rename from src/services/impersonation_token/optype.ts rename to app/sdms/src/services/impersonation_token/optype.ts diff --git a/src/services/impersonation_token/parser.ts b/app/sdms/src/services/impersonation_token/parser.ts similarity index 100% rename from src/services/impersonation_token/parser.ts rename to app/sdms/src/services/impersonation_token/parser.ts diff --git a/src/services/impersonation_token/service.ts b/app/sdms/src/services/impersonation_token/service.ts similarity index 100% rename from src/services/impersonation_token/service.ts rename to app/sdms/src/services/impersonation_token/service.ts diff --git a/src/services/imptoken/dao.ts b/app/sdms/src/services/imptoken/dao.ts similarity index 100% rename from src/services/imptoken/dao.ts rename to app/sdms/src/services/imptoken/dao.ts diff --git a/src/services/imptoken/handler.ts b/app/sdms/src/services/imptoken/handler.ts similarity index 100% rename from src/services/imptoken/handler.ts rename to app/sdms/src/services/imptoken/handler.ts diff --git a/src/services/imptoken/index.ts b/app/sdms/src/services/imptoken/index.ts similarity index 100% rename from src/services/imptoken/index.ts rename to app/sdms/src/services/imptoken/index.ts diff --git a/src/services/imptoken/model.ts b/app/sdms/src/services/imptoken/model.ts similarity index 100% rename from src/services/imptoken/model.ts rename to app/sdms/src/services/imptoken/model.ts diff --git a/src/services/imptoken/optype.ts b/app/sdms/src/services/imptoken/optype.ts similarity index 100% rename from src/services/imptoken/optype.ts rename to app/sdms/src/services/imptoken/optype.ts diff --git a/src/services/imptoken/parser.ts b/app/sdms/src/services/imptoken/parser.ts similarity index 100% rename from src/services/imptoken/parser.ts rename to app/sdms/src/services/imptoken/parser.ts diff --git a/src/services/imptoken/service.ts b/app/sdms/src/services/imptoken/service.ts similarity index 100% rename from src/services/imptoken/service.ts rename to app/sdms/src/services/imptoken/service.ts diff --git a/src/services/index.ts b/app/sdms/src/services/index.ts similarity index 100% rename from src/services/index.ts rename to app/sdms/src/services/index.ts diff --git a/src/services/subproject/dao.ts b/app/sdms/src/services/subproject/dao.ts similarity index 100% rename from src/services/subproject/dao.ts rename to app/sdms/src/services/subproject/dao.ts diff --git a/src/services/subproject/groups.ts b/app/sdms/src/services/subproject/groups.ts similarity index 100% rename from src/services/subproject/groups.ts rename to app/sdms/src/services/subproject/groups.ts diff --git a/src/services/subproject/handler.ts b/app/sdms/src/services/subproject/handler.ts similarity index 100% rename from src/services/subproject/handler.ts rename to app/sdms/src/services/subproject/handler.ts diff --git a/src/services/subproject/index.ts b/app/sdms/src/services/subproject/index.ts similarity index 100% rename from src/services/subproject/index.ts rename to app/sdms/src/services/subproject/index.ts diff --git a/src/services/subproject/model.ts b/app/sdms/src/services/subproject/model.ts similarity index 100% rename from src/services/subproject/model.ts rename to app/sdms/src/services/subproject/model.ts diff --git a/src/services/subproject/optype.ts b/app/sdms/src/services/subproject/optype.ts similarity index 100% rename from src/services/subproject/optype.ts rename to app/sdms/src/services/subproject/optype.ts diff --git a/src/services/subproject/parser.ts b/app/sdms/src/services/subproject/parser.ts similarity index 100% rename from src/services/subproject/parser.ts rename to app/sdms/src/services/subproject/parser.ts diff --git a/src/services/subproject/service.ts b/app/sdms/src/services/subproject/service.ts similarity index 100% rename from src/services/subproject/service.ts rename to app/sdms/src/services/subproject/service.ts diff --git a/src/services/svcapp/dao.ts b/app/sdms/src/services/svcapp/dao.ts similarity index 100% rename from src/services/svcapp/dao.ts rename to app/sdms/src/services/svcapp/dao.ts diff --git a/src/services/svcapp/handler.ts b/app/sdms/src/services/svcapp/handler.ts similarity index 100% rename from src/services/svcapp/handler.ts rename to app/sdms/src/services/svcapp/handler.ts diff --git a/src/services/svcapp/model.ts b/app/sdms/src/services/svcapp/model.ts similarity index 100% rename from src/services/svcapp/model.ts rename to app/sdms/src/services/svcapp/model.ts diff --git a/src/services/svcapp/optype.ts b/app/sdms/src/services/svcapp/optype.ts similarity index 100% rename from src/services/svcapp/optype.ts rename to app/sdms/src/services/svcapp/optype.ts diff --git a/src/services/svcapp/parser.ts b/app/sdms/src/services/svcapp/parser.ts similarity index 100% rename from src/services/svcapp/parser.ts rename to app/sdms/src/services/svcapp/parser.ts diff --git a/src/services/svcapp/service.ts b/app/sdms/src/services/svcapp/service.ts similarity index 100% rename from src/services/svcapp/service.ts rename to app/sdms/src/services/svcapp/service.ts diff --git a/src/services/tenant/dao.ts b/app/sdms/src/services/tenant/dao.ts similarity index 100% rename from src/services/tenant/dao.ts rename to app/sdms/src/services/tenant/dao.ts diff --git a/src/services/tenant/groups.ts b/app/sdms/src/services/tenant/groups.ts similarity index 100% rename from src/services/tenant/groups.ts rename to app/sdms/src/services/tenant/groups.ts diff --git a/src/services/tenant/handler.ts b/app/sdms/src/services/tenant/handler.ts similarity index 100% rename from src/services/tenant/handler.ts rename to app/sdms/src/services/tenant/handler.ts diff --git a/src/services/tenant/index.ts b/app/sdms/src/services/tenant/index.ts similarity index 100% rename from src/services/tenant/index.ts rename to app/sdms/src/services/tenant/index.ts diff --git a/src/services/tenant/model.ts b/app/sdms/src/services/tenant/model.ts similarity index 100% rename from src/services/tenant/model.ts rename to app/sdms/src/services/tenant/model.ts diff --git a/src/services/tenant/optype.ts b/app/sdms/src/services/tenant/optype.ts similarity index 100% rename from src/services/tenant/optype.ts rename to app/sdms/src/services/tenant/optype.ts diff --git a/src/services/tenant/parser.ts b/app/sdms/src/services/tenant/parser.ts similarity index 100% rename from src/services/tenant/parser.ts rename to app/sdms/src/services/tenant/parser.ts diff --git a/src/services/tenant/service.ts b/app/sdms/src/services/tenant/service.ts similarity index 100% rename from src/services/tenant/service.ts rename to app/sdms/src/services/tenant/service.ts diff --git a/src/services/user/handler.ts b/app/sdms/src/services/user/handler.ts similarity index 100% rename from src/services/user/handler.ts rename to app/sdms/src/services/user/handler.ts diff --git a/src/services/user/optype.ts b/app/sdms/src/services/user/optype.ts similarity index 100% rename from src/services/user/optype.ts rename to app/sdms/src/services/user/optype.ts diff --git a/src/services/user/parser.ts b/app/sdms/src/services/user/parser.ts similarity index 100% rename from src/services/user/parser.ts rename to app/sdms/src/services/user/parser.ts diff --git a/src/services/user/service.ts b/app/sdms/src/services/user/service.ts similarity index 100% rename from src/services/user/service.ts rename to app/sdms/src/services/user/service.ts diff --git a/src/services/utility/handler.ts b/app/sdms/src/services/utility/handler.ts similarity index 100% rename from src/services/utility/handler.ts rename to app/sdms/src/services/utility/handler.ts diff --git a/src/services/utility/optype.ts b/app/sdms/src/services/utility/optype.ts similarity index 100% rename from src/services/utility/optype.ts rename to app/sdms/src/services/utility/optype.ts diff --git a/src/services/utility/parser.ts b/app/sdms/src/services/utility/parser.ts similarity index 100% rename from src/services/utility/parser.ts rename to app/sdms/src/services/utility/parser.ts diff --git a/src/services/utility/service.ts b/app/sdms/src/services/utility/service.ts similarity index 100% rename from src/services/utility/service.ts rename to app/sdms/src/services/utility/service.ts diff --git a/src/shared/cache.ts b/app/sdms/src/shared/cache.ts similarity index 100% rename from src/shared/cache.ts rename to app/sdms/src/shared/cache.ts diff --git a/src/shared/error.ts b/app/sdms/src/shared/error.ts similarity index 100% rename from src/shared/error.ts rename to app/sdms/src/shared/error.ts diff --git a/src/shared/featureflags.ts b/app/sdms/src/shared/featureflags.ts similarity index 100% rename from src/shared/featureflags.ts rename to app/sdms/src/shared/featureflags.ts diff --git a/src/shared/index.ts b/app/sdms/src/shared/index.ts similarity index 100% rename from src/shared/index.ts rename to app/sdms/src/shared/index.ts diff --git a/src/shared/params.ts b/app/sdms/src/shared/params.ts similarity index 100% rename from src/shared/params.ts rename to app/sdms/src/shared/params.ts diff --git a/src/shared/response.ts b/app/sdms/src/shared/response.ts similarity index 100% rename from src/shared/response.ts rename to app/sdms/src/shared/response.ts diff --git a/src/shared/sdpath.ts b/app/sdms/src/shared/sdpath.ts similarity index 100% rename from src/shared/sdpath.ts rename to app/sdms/src/shared/sdpath.ts diff --git a/src/shared/tracelog.ts b/app/sdms/src/shared/tracelog.ts similarity index 100% rename from src/shared/tracelog.ts rename to app/sdms/src/shared/tracelog.ts diff --git a/src/shared/utils.ts b/app/sdms/src/shared/utils.ts similarity index 100% rename from src/shared/utils.ts rename to app/sdms/src/shared/utils.ts diff --git a/tests/aws-test/build-aws/prepare-dist.sh b/app/sdms/tests/aws-test/build-aws/prepare-dist.sh old mode 100755 new mode 100644 similarity index 100% rename from tests/aws-test/build-aws/prepare-dist.sh rename to app/sdms/tests/aws-test/build-aws/prepare-dist.sh diff --git a/tests/aws-test/build-aws/run-tests.sh b/app/sdms/tests/aws-test/build-aws/run-tests.sh old mode 100755 new mode 100644 similarity index 100% rename from tests/aws-test/build-aws/run-tests.sh rename to app/sdms/tests/aws-test/build-aws/run-tests.sh diff --git a/tests/e2e/parallel/postman_collection.dataset_delete.json b/app/sdms/tests/e2e/parallel/postman_collection.dataset_delete.json similarity index 100% rename from tests/e2e/parallel/postman_collection.dataset_delete.json rename to app/sdms/tests/e2e/parallel/postman_collection.dataset_delete.json diff --git a/tests/e2e/parallel/postman_collection.dataset_readlock.json b/app/sdms/tests/e2e/parallel/postman_collection.dataset_readlock.json similarity index 100% rename from tests/e2e/parallel/postman_collection.dataset_readlock.json rename to app/sdms/tests/e2e/parallel/postman_collection.dataset_readlock.json diff --git a/tests/e2e/parallel/postman_collection.dataset_register.json b/app/sdms/tests/e2e/parallel/postman_collection.dataset_register.json similarity index 100% rename from tests/e2e/parallel/postman_collection.dataset_register.json rename to app/sdms/tests/e2e/parallel/postman_collection.dataset_register.json diff --git a/tests/e2e/parallel/postman_collection.dataset_unlock.json b/app/sdms/tests/e2e/parallel/postman_collection.dataset_unlock.json similarity index 100% rename from tests/e2e/parallel/postman_collection.dataset_unlock.json rename to app/sdms/tests/e2e/parallel/postman_collection.dataset_unlock.json diff --git a/tests/e2e/parallel/postman_collection.dataset_writelock.json b/app/sdms/tests/e2e/parallel/postman_collection.dataset_writelock.json similarity index 100% rename from tests/e2e/parallel/postman_collection.dataset_writelock.json rename to app/sdms/tests/e2e/parallel/postman_collection.dataset_writelock.json diff --git a/tests/e2e/parallel/postman_collection.gcstoken.json b/app/sdms/tests/e2e/parallel/postman_collection.gcstoken.json similarity index 100% rename from tests/e2e/parallel/postman_collection.gcstoken.json rename to app/sdms/tests/e2e/parallel/postman_collection.gcstoken.json diff --git a/tests/e2e/parallel/postman_env.json b/app/sdms/tests/e2e/parallel/postman_env.json similarity index 100% rename from tests/e2e/parallel/postman_env.json rename to app/sdms/tests/e2e/parallel/postman_env.json diff --git a/tests/e2e/parallel/run_e2e_tests_parallel.js b/app/sdms/tests/e2e/parallel/run_e2e_tests_parallel.js similarity index 100% rename from tests/e2e/parallel/run_e2e_tests_parallel.js rename to app/sdms/tests/e2e/parallel/run_e2e_tests_parallel.js diff --git a/tests/e2e/parallel/run_e2e_tests_parallel.sh b/app/sdms/tests/e2e/parallel/run_e2e_tests_parallel.sh similarity index 100% rename from tests/e2e/parallel/run_e2e_tests_parallel.sh rename to app/sdms/tests/e2e/parallel/run_e2e_tests_parallel.sh diff --git a/tests/e2e/postman_collection.json b/app/sdms/tests/e2e/postman_collection.json similarity index 100% rename from tests/e2e/postman_collection.json rename to app/sdms/tests/e2e/postman_collection.json diff --git a/tests/e2e/postman_env.json b/app/sdms/tests/e2e/postman_env.json similarity index 100% rename from tests/e2e/postman_env.json rename to app/sdms/tests/e2e/postman_env.json diff --git a/tests/e2e/run_e2e_tests.sh b/app/sdms/tests/e2e/run_e2e_tests.sh old mode 100755 new mode 100644 similarity index 100% rename from tests/e2e/run_e2e_tests.sh rename to app/sdms/tests/e2e/run_e2e_tests.sh diff --git a/tests/utest/auth/auth.ts b/app/sdms/tests/utest/auth/auth.ts similarity index 100% rename from tests/utest/auth/auth.ts rename to app/sdms/tests/utest/auth/auth.ts diff --git a/tests/utest/auth/groups.ts b/app/sdms/tests/utest/auth/groups.ts similarity index 100% rename from tests/utest/auth/groups.ts rename to app/sdms/tests/utest/auth/groups.ts diff --git a/tests/utest/auth/test.ts b/app/sdms/tests/utest/auth/test.ts similarity index 100% rename from tests/utest/auth/test.ts rename to app/sdms/tests/utest/auth/test.ts diff --git a/tests/utest/cloud/azure/cloudstorage.ts b/app/sdms/tests/utest/cloud/azure/cloudstorage.ts similarity index 100% rename from tests/utest/cloud/azure/cloudstorage.ts rename to app/sdms/tests/utest/cloud/azure/cloudstorage.ts diff --git a/tests/utest/cloud/azure/cosmosdb-query.ts b/app/sdms/tests/utest/cloud/azure/cosmosdb-query.ts similarity index 100% rename from tests/utest/cloud/azure/cosmosdb-query.ts rename to app/sdms/tests/utest/cloud/azure/cosmosdb-query.ts diff --git a/tests/utest/cloud/azure/cosmosdb-transactions.ts b/app/sdms/tests/utest/cloud/azure/cosmosdb-transactions.ts similarity index 100% rename from tests/utest/cloud/azure/cosmosdb-transactions.ts rename to app/sdms/tests/utest/cloud/azure/cosmosdb-transactions.ts diff --git a/tests/utest/cloud/azure/cosmosdb.ts b/app/sdms/tests/utest/cloud/azure/cosmosdb.ts similarity index 100% rename from tests/utest/cloud/azure/cosmosdb.ts rename to app/sdms/tests/utest/cloud/azure/cosmosdb.ts diff --git a/tests/utest/cloud/azure/keyvault.ts b/app/sdms/tests/utest/cloud/azure/keyvault.ts similarity index 100% rename from tests/utest/cloud/azure/keyvault.ts rename to app/sdms/tests/utest/cloud/azure/keyvault.ts diff --git a/tests/utest/cloud/credentials.ts b/app/sdms/tests/utest/cloud/credentials.ts similarity index 100% rename from tests/utest/cloud/credentials.ts rename to app/sdms/tests/utest/cloud/credentials.ts diff --git a/tests/utest/cloud/datastore.ts b/app/sdms/tests/utest/cloud/datastore.ts similarity index 100% rename from tests/utest/cloud/datastore.ts rename to app/sdms/tests/utest/cloud/datastore.ts diff --git a/tests/utest/cloud/gcs.ts b/app/sdms/tests/utest/cloud/gcs.ts similarity index 100% rename from tests/utest/cloud/gcs.ts rename to app/sdms/tests/utest/cloud/gcs.ts diff --git a/tests/utest/cloud/test.ts b/app/sdms/tests/utest/cloud/test.ts similarity index 100% rename from tests/utest/cloud/test.ts rename to app/sdms/tests/utest/cloud/test.ts diff --git a/tests/utest/dao/dataset.ts b/app/sdms/tests/utest/dao/dataset.ts similarity index 100% rename from tests/utest/dao/dataset.ts rename to app/sdms/tests/utest/dao/dataset.ts diff --git a/tests/utest/dao/imptoken.ts b/app/sdms/tests/utest/dao/imptoken.ts similarity index 100% rename from tests/utest/dao/imptoken.ts rename to app/sdms/tests/utest/dao/imptoken.ts diff --git a/tests/utest/dao/subproject.ts b/app/sdms/tests/utest/dao/subproject.ts similarity index 100% rename from tests/utest/dao/subproject.ts rename to app/sdms/tests/utest/dao/subproject.ts diff --git a/tests/utest/dao/svcapp.ts b/app/sdms/tests/utest/dao/svcapp.ts similarity index 100% rename from tests/utest/dao/svcapp.ts rename to app/sdms/tests/utest/dao/svcapp.ts diff --git a/tests/utest/dao/tenant.ts b/app/sdms/tests/utest/dao/tenant.ts similarity index 100% rename from tests/utest/dao/tenant.ts rename to app/sdms/tests/utest/dao/tenant.ts diff --git a/tests/utest/dao/test.ts b/app/sdms/tests/utest/dao/test.ts similarity index 100% rename from tests/utest/dao/test.ts rename to app/sdms/tests/utest/dao/test.ts diff --git a/tests/utest/dataecosystem/compliance.ts b/app/sdms/tests/utest/dataecosystem/compliance.ts similarity index 100% rename from tests/utest/dataecosystem/compliance.ts rename to app/sdms/tests/utest/dataecosystem/compliance.ts diff --git a/tests/utest/dataecosystem/entitlement.ts b/app/sdms/tests/utest/dataecosystem/entitlement.ts similarity index 100% rename from tests/utest/dataecosystem/entitlement.ts rename to app/sdms/tests/utest/dataecosystem/entitlement.ts diff --git a/tests/utest/dataecosystem/storage.ts b/app/sdms/tests/utest/dataecosystem/storage.ts similarity index 100% rename from tests/utest/dataecosystem/storage.ts rename to app/sdms/tests/utest/dataecosystem/storage.ts diff --git a/tests/utest/dataecosystem/test.ts b/app/sdms/tests/utest/dataecosystem/test.ts similarity index 100% rename from tests/utest/dataecosystem/test.ts rename to app/sdms/tests/utest/dataecosystem/test.ts diff --git a/tests/utest/dataecosystem/utils.ts b/app/sdms/tests/utest/dataecosystem/utils.ts similarity index 100% rename from tests/utest/dataecosystem/utils.ts rename to app/sdms/tests/utest/dataecosystem/utils.ts diff --git a/tests/utest/services/app.ts b/app/sdms/tests/utest/services/app.ts similarity index 100% rename from tests/utest/services/app.ts rename to app/sdms/tests/utest/services/app.ts diff --git a/tests/utest/services/dataset.ts b/app/sdms/tests/utest/services/dataset.ts similarity index 100% rename from tests/utest/services/dataset.ts rename to app/sdms/tests/utest/services/dataset.ts diff --git a/tests/utest/services/general.ts b/app/sdms/tests/utest/services/general.ts similarity index 100% rename from tests/utest/services/general.ts rename to app/sdms/tests/utest/services/general.ts diff --git a/tests/utest/services/imptoken.ts b/app/sdms/tests/utest/services/imptoken.ts similarity index 100% rename from tests/utest/services/imptoken.ts rename to app/sdms/tests/utest/services/imptoken.ts diff --git a/tests/utest/services/locker.ts b/app/sdms/tests/utest/services/locker.ts similarity index 100% rename from tests/utest/services/locker.ts rename to app/sdms/tests/utest/services/locker.ts diff --git a/tests/utest/services/subproject.ts b/app/sdms/tests/utest/services/subproject.ts similarity index 100% rename from tests/utest/services/subproject.ts rename to app/sdms/tests/utest/services/subproject.ts diff --git a/tests/utest/services/tenant.ts b/app/sdms/tests/utest/services/tenant.ts similarity index 100% rename from tests/utest/services/tenant.ts rename to app/sdms/tests/utest/services/tenant.ts diff --git a/tests/utest/services/test.ts b/app/sdms/tests/utest/services/test.ts similarity index 100% rename from tests/utest/services/test.ts rename to app/sdms/tests/utest/services/test.ts diff --git a/tests/utest/services/user.ts b/app/sdms/tests/utest/services/user.ts similarity index 100% rename from tests/utest/services/user.ts rename to app/sdms/tests/utest/services/user.ts diff --git a/tests/utest/services/utility.ts b/app/sdms/tests/utest/services/utility.ts similarity index 100% rename from tests/utest/services/utility.ts rename to app/sdms/tests/utest/services/utility.ts diff --git a/tests/utest/shared/error.ts b/app/sdms/tests/utest/shared/error.ts similarity index 100% rename from tests/utest/shared/error.ts rename to app/sdms/tests/utest/shared/error.ts diff --git a/tests/utest/shared/logger.ts b/app/sdms/tests/utest/shared/logger.ts similarity index 100% rename from tests/utest/shared/logger.ts rename to app/sdms/tests/utest/shared/logger.ts diff --git a/tests/utest/shared/params.ts b/app/sdms/tests/utest/shared/params.ts similarity index 100% rename from tests/utest/shared/params.ts rename to app/sdms/tests/utest/shared/params.ts diff --git a/tests/utest/shared/response.ts b/app/sdms/tests/utest/shared/response.ts similarity index 100% rename from tests/utest/shared/response.ts rename to app/sdms/tests/utest/shared/response.ts diff --git a/tests/utest/shared/sdpath.ts b/app/sdms/tests/utest/shared/sdpath.ts similarity index 100% rename from tests/utest/shared/sdpath.ts rename to app/sdms/tests/utest/shared/sdpath.ts diff --git a/tests/utest/shared/test.ts b/app/sdms/tests/utest/shared/test.ts similarity index 100% rename from tests/utest/shared/test.ts rename to app/sdms/tests/utest/shared/test.ts diff --git a/tests/utest/shared/utils.ts b/app/sdms/tests/utest/shared/utils.ts similarity index 100% rename from tests/utest/shared/utils.ts rename to app/sdms/tests/utest/shared/utils.ts diff --git a/tests/utest/test.ts b/app/sdms/tests/utest/test.ts similarity index 100% rename from tests/utest/test.ts rename to app/sdms/tests/utest/test.ts diff --git a/tests/utest/utils.ts b/app/sdms/tests/utest/utils.ts similarity index 100% rename from tests/utest/utils.ts rename to app/sdms/tests/utest/utils.ts diff --git a/tsconfig.json b/app/sdms/tsconfig.json similarity index 100% rename from tsconfig.json rename to app/sdms/tsconfig.json diff --git a/tslint.json b/app/sdms/tslint.json similarity index 100% rename from tslint.json rename to app/sdms/tslint.json -- GitLab From 072ff2176832614af7db72d90e80affb928d2948 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 08:41:32 -0500 Subject: [PATCH 02/45] ci: testing trigger in build --- app/sdms/devops/osdu/build/seismic-store-service.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/sdms/devops/osdu/build/seismic-store-service.yml b/app/sdms/devops/osdu/build/seismic-store-service.yml index a7a8de35..4ac94f41 100644 --- a/app/sdms/devops/osdu/build/seismic-store-service.yml +++ b/app/sdms/devops/osdu/build/seismic-store-service.yml @@ -14,4 +14,4 @@ compile-and-unit-test: - devops/scripts/utest_build_run.sh only: changes: - - app/sdms + - app/sdms/**/* -- GitLab From 10f42772485e91295948c5d266effd4ac9e83add Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 08:51:28 -0500 Subject: [PATCH 03/45] ci: updating sdms build and scanner steps --- app/sdms/.gitlab-ci.yml | 12 ++++++++++++ app/sdms/devops/osdu/build/seismic-store-service.yml | 1 + app/sdms/devops/osdu/scanners/fossa-node.yml | 10 ++++++---- app/sdms/devops/osdu/scanners/lint-node.yml | 6 +++++- .../devops/osdu/scanners/scan-for-secrets-node.yml | 6 +++++- 5 files changed, 29 insertions(+), 6 deletions(-) diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml index 895b8e70..dd53cb26 100644 --- a/app/sdms/.gitlab-ci.yml +++ b/app/sdms/.gitlab-ci.yml @@ -46,3 +46,15 @@ include: # build - local: "app/sdms/devops/osdu/build/seismic-store-service.yml" + + # scan + # fossa + - local: "app/sdms/devops/osdu/scanners/fossa-node.yml" + + # lint + - local: "app/sdms/devops/osdu/scanners/lint-node.yml" + + # scan for secrets + - local: "app/sdms/devops/osdu/scanners/scan-for-secrets-node.yml" + + \ No newline at end of file diff --git a/app/sdms/devops/osdu/build/seismic-store-service.yml b/app/sdms/devops/osdu/build/seismic-store-service.yml index 4ac94f41..0aaf98f7 100644 --- a/app/sdms/devops/osdu/build/seismic-store-service.yml +++ b/app/sdms/devops/osdu/build/seismic-store-service.yml @@ -10,6 +10,7 @@ compile-and-unit-test: - coverage/ expire_in: 15 days script: + - cd app/sdms - chmod +x devops/scripts/utest_build_run.sh - devops/scripts/utest_build_run.sh only: diff --git a/app/sdms/devops/osdu/scanners/fossa-node.yml b/app/sdms/devops/osdu/scanners/fossa-node.yml index 80f4c6a3..2dac2201 100644 --- a/app/sdms/devops/osdu/scanners/fossa-node.yml +++ b/app/sdms/devops/osdu/scanners/fossa-node.yml @@ -9,15 +9,17 @@ fossa-analyze: stage: scan needs: ['compile-and-unit-test'] tags: ['osdu-medium'] - only: - variables: - - $FOSSA_API_KEY script: # fossa-check-for-licensing-issues needs a CI_COMMIT_BRANCH defined to know how to parse the FOSSA API results # When building tags, this isn't defined by GitLab. In that case, we use the tag name instead. If that's not defined # then things will fail and we'll have to make this smarter + - cd app/sdms - test -z "$CI_COMMIT_BRANCH" && export CI_COMMIT_BRANCH="$CI_COMMIT_TAG" - - npm ci --production - fossa analyze --project "${CI_PROJECT_TITLE}" --project-url "${CI_PROJECT_URL}" --branch "${CI_COMMIT_BRANCH}" - fossa-check-for-licensing-issues + only: + variables: + - $FOSSA_API_KEY + changes: + - app/sdms/**/* diff --git a/app/sdms/devops/osdu/scanners/lint-node.yml b/app/sdms/devops/osdu/scanners/lint-node.yml index d79017d4..993fa275 100644 --- a/app/sdms/devops/osdu/scanners/lint-node.yml +++ b/app/sdms/devops/osdu/scanners/lint-node.yml @@ -4,6 +4,10 @@ lint: stage: scan needs: ['compile-and-unit-test'] script: + - cd app/sdms - npm install -g typescript - npm install -g tslint - - npm run lint \ No newline at end of file + - npm run lint + only: + changes: + - app/sdms/**/* \ No newline at end of file diff --git a/app/sdms/devops/osdu/scanners/scan-for-secrets-node.yml b/app/sdms/devops/osdu/scanners/scan-for-secrets-node.yml index 0ceeaca4..4d824c56 100644 --- a/app/sdms/devops/osdu/scanners/scan-for-secrets-node.yml +++ b/app/sdms/devops/osdu/scanners/scan-for-secrets-node.yml @@ -6,4 +6,8 @@ scan-for-secrets: - job: compile-and-unit-test artifacts: false script: - - detect-secrets-hook --exclude-files npm-shrinkwrap.json --exclude-files package.json --exclude-files devops/osdu/scanners/scan-for-secrets-node.yml --baseline devops/docker/detect_secrets/.secrets.baseline $(git ls-files) \ No newline at end of file + - cd app/sdms + - detect-secrets-hook --exclude-files npm-shrinkwrap.json --exclude-files package.json --exclude-files devops/osdu/scanners/scan-for-secrets-node.yml --baseline devops/docker/detect_secrets/.secrets.baseline $(git ls-files) + only: + changes: + - app/sdms/**/* \ No newline at end of file -- GitLab From 898d75fea5fa57ea8cfb3a6018bf523674d6f2c4 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 09:55:54 -0500 Subject: [PATCH 04/45] ci: uodating contenerize and integration steps --- app/sdms/.gitlab-ci.yml | 22 +- .../aws-seismic-store-service.yml | 229 ++++++++++++++++++ .../azure-seismic-store-service.yml | 24 ++ .../contenerize/gcp-seismic-store-service.yml | 19 ++ .../contenerize/seismic-store-service.yml | 23 ++ .../{aws => osdu/integration}/awstest.yml | 0 .../integration}/requirements.txt | 0 7 files changed, 316 insertions(+), 1 deletion(-) create mode 100644 app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml create mode 100644 app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml create mode 100644 app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml create mode 100644 app/sdms/devops/osdu/contenerize/seismic-store-service.yml rename app/sdms/devops/{aws => osdu/integration}/awstest.yml (100%) rename app/sdms/devops/{aws => osdu/integration}/requirements.txt (100%) diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml index dd53cb26..bc36a50e 100644 --- a/app/sdms/.gitlab-ci.yml +++ b/app/sdms/.gitlab-ci.yml @@ -57,4 +57,24 @@ include: # scan for secrets - local: "app/sdms/devops/osdu/scanners/scan-for-secrets-node.yml" - \ No newline at end of file + # containerize + # osdu + - project: "app/sdms/devops/osdu/contenerize/seismic-store-service.yml" + + # azure + - project: "app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml" + + # gcp + - project: "app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml" + + # deploy + # azure + - local: "app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml" + + # aws + - local: "app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml" + + # integration + # aws + - local: "app/sdms/devops/osdu/integration/awstest.yml" + diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml new file mode 100644 index 00000000..f5323508 --- /dev/null +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -0,0 +1,229 @@ +.aws_variables: + + before_script: + - | + if [ $AWS_DEPLOY_TARGET == 'EKS' ]; then + + echo "##### DEPLOYING TO EKS/GREEN ######" + + export AWS_INFRA_STACK_NAME="$AWS_GREEN_INFRA_STACK_NAME"; + export AWS_REGION="$AWS_GREEN_REGION"; + export AWS_DOMAIN="${AWS_GREEN_SUBDOMAIN}.${AWS_BASE_DOMAIN}"; + export RESOURCE_PREFIX="$AWS_GREEN_RESOURCE_PREFIX"; + + else + + echo "##### DEPLOYING TO ECS/BLUE ######" + + echo "FAILED>>>BLUE ENVIRONMENT DOES NOT EXIST" + + exit 1 + + export AWS_INFRA_STACK_NAME="$AWS_BLUE_INFRA_STACK_NAME"; + export AWS_REGION="$AWS_BLUE_REGION"; + export AWS_DOMAIN="${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}"; + export RESOURCE_PREFIX="$AWS_BLUE_RESOURCE_PREFIX"; + + fi + + export S3_LEGAL_CONFIG_BUCKET=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/shared/legal/s3DataBucket/name --query Parameter.Value --output text --region $AWS_REGION); + export AWS_COGNITO_CLIENT_ID=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/cognito-client-id --query Parameter.Value --output text --region $AWS_REGION); + export AWS_COGNITO_USER_POOL_ID=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/cognito-user-pool-id --query Parameter.Value --output text --region $AWS_REGION); + + export ELASTIC_HOST=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/elasticsearch/end-point --query Parameter.Value --output text --region $AWS_REGION); + export ELASTIC_SECRET_STRING=$(aws secretsmanager get-secret-value --secret-id /osdu/${RESOURCE_PREFIX}/elasticsearch/credentials --query SecretString --output json --region $AWS_REGION | sed -e 's/\\\"/\"/g' -e 's/^.//g' -e 's/.$//g'); + export ELASTIC_USER_NAME=$(echo $ELASTIC_SECRET_STRING | jq -r '.username'); + export ELASTIC_PASSWORD=$(echo $ELASTIC_SECRET_STRING | jq -r '.password'); + + export AWS_BASE_URL="https://${AWS_DOMAIN}"; + export HOST_URL="${AWS_BASE_URL}/api/legal/v1/" + export LEGALTAG_BASE_URL="${AWS_DOMAIN}" + export STORAGE_URL="${AWS_BASE_URL}/api/storage/v2/" + export LEGAL_URL="${AWS_BASE_URL}/api/legal/v1/" + export SEARCH_HOST="${AWS_BASE_URL}/api/search/v2/" + export STORAGE_HOST="${AWS_BASE_URL}/api/storage/v2/" + export LEGAL_HOST="${AWS_BASE_URL}/api/legal/v1/" + export DELIVERY_HOST="${AWS_BASE_URL}/api/delivery/v2/" + export INDEXER_HOST="${AWS_BASE_URL}/api/indexer/v2/" + export ENTITLEMENTS_URL="${AWS_BASE_URL}/api/entitlements/v2/" + export VIRTUAL_SERVICE_HOST_NAME="${AWS_DOMAIN}" + export HOST="${AWS_BASE_URL}" + # needs trailing slash + export PARTITION_BASE_URL="${AWS_BASE_URL}/" + export WORKFLOW_HOST="${AWS_BASE_URL}/api/workflow/" + export DATA_WORKFLOW_HOST="${AWS_BASE_URL}/api/data-workflow/v1" + export REGISTER_CUSTOM_PUSH_URL="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge" + export REGISTER_CUSTOM_PUSH_URL1="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge/1" + # needs trailing slash + export REGISTER_BASE_URL="${AWS_BASE_URL}/" + export REGISTER_CUSTOM_PUSH_URL_HMAC="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge/1" + export NOTIFICATION_BASE_URL="${AWS_BASE_URL}/api/notification/v1/" + export NOTIFICATION_REGISTER_BASE_URL="${AWS_BASE_URL}" + export FILE_SERVICE_HOST="${AWS_BASE_URL}/api/file/v2" + export STORAGE_BASE_URL="${AWS_BASE_URL}/api/storage/v2/" + export DATASET_BASE_URL="${AWS_BASE_URL}/api/dataset/v1/" + export LEGAL_BASE_URL="${AWS_BASE_URL}/api/legal/v1/" + export ENTITLEMENTS_BASE_URL="${AWS_BASE_URL}/api/entitlements/v2/" + export FILEDMS_BASE_URL="${AWS_BASE_URL}/api/dms/file/v1/" + export SEISMICSTORE_SVC_URL="${AWS_BASE_URL}/api/seismic-store/v3" + export WELLBORE_DDMS_URL="${AWS_BASE_URL}/api/os-wellbore-ddms" + export EDSDMS_BASE_URL="${AWS_BASE_URL}/api/dms/eds/v1/" + export SCHEMA_BASE_URL="${AWS_BASE_URL}/api/schema-service/v1/" + export ENTITLEMENT_V2_URL="${AWS_BASE_URL}/api/entitlements/v2/" + export LEGAL_QUEUE="https://sqs.${AWS_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${RESOURCE_PREFIX}-legal-queue" + + export AWS_COGNITO_REGION="$AWS_REGION" + export AWS_S3_ENDPOINT="s3.${AWS_REGION}.amazonaws.com" + export AWS_S3_REGION="$AWS_REGION" + + export DYNAMO_DB_REGION="$AWS_REGION" + export DYNAMO_DB_ENDPOINT="dynamodb.${AWS_REGION}.amazonaws.com" + + export TABLE_PREFIX="$RESOURCE_PREFIX" + export AWS_CLUSTER_NAME="${RESOURCE_PREFIX}-core-cluster" + + echo "#### USING OSDU BASE URL: $AWS_BASE_URL ####" + +.aws_common_variables: + variables: + ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID + SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY + AWS_ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY + INTEGRATION_TEST_DIR: $AWS_TEST_SUBDIR + SERVICE_NAME: $AWS_SERVICE + BUILD_DIR: $AWS_BUILD_SUBDIR + ENVIRONMENT: $AWS_ENVIRONMENT + APPLICATION_NAME: os-$AWS_SERVICE + LOCAL_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA + AWS_IMAGE_TAG_BASE: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_ECR_REGION}.amazonaws.com/os-${AWS_SERVICE} + OTHER_RELEVANT_DATA_COUNTRIES: $AWS_OTHER_RELEVANT_DATA_COUNTRIES + LEGAL_TAG : $AWS_LEGAL_TAG + TENANT_NAME : $AWS_TENANT_NAME + PRIVATE_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1 + PRIVATE_TENANT2: tenant2 + SHARED_TENANT: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2 + VENDOR: aws + DOMAIN: example.com + AWS_COGNITO_AUTH_FLOW: $AWS_COGNITO_AUTH_FLOW + AWS_COGNITO_AUTH_PARAMS_PASSWORD: $AWS_COGNITO_AUTH_PARAMS_PASSWORD + AWS_COGNITO_AUTH_PARAMS_USER: $AWS_COGNITO_AUTH_PARAMS_USER + AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS: $AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS + AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID + DEPLOY_ENV: empty + MY_TENANT: $AWS_TENANT_NAME + ENTITLEMENTS_TEST_TENANT: $AWS_ENTITLEMENTS_TEST_TENANT + LOG_LEVEL: INFO + SKIP_HTTP_TESTS: 'true' + DELIVERY_INT_TEST_BUCKET_NAME: $AWS_DELIVERY_INT_TEST_BUCKET_NAME + DEFAULT_DATA_PARTITION_ID_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1 + DEFAULT_DATA_PARTITION_ID_TENANT2: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2 + #used by file->delivery int tests + DATA_PARTITION_ID: int-test-file + ENTITLEMENTS_DOMAIN: example.com + ELASTIC_PORT: 9200 + TEST_DAG_NAME: my_first_dag + SUBSCRIBER_SECRET: $AWS_SUBSCRIBER_SECRET + HMAC_SECRET: $AWS_HMAC_SECRET + PROVIDER_KEY: AWS_S3 + #File Service Variables + TIME_ZONE: UTC + USER_ID: $AWS_COGNITO_AUTH_PARAMS_USER + #Entitlements V2 variables--start-- + TENANT: opendes + SERVICE_PRINCIPAL_EMAIL: serviceprincipal@testing.com + #Entitlements V2 variables--end-- + + +.aws: + tags: ['osdu-medium'] + image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-maven/aws-maven:v1.3 + environment: + name: AWS + extends: + - .aws_common_variables + - .aws_variables + before_script: + - mkdir -p ~/.aws + - | + cat > ~/.aws/credentials <RUNTIME containers; + docker build -t builder:latest --file $AWS_BUILDER_DOCKERFILE_PATH .; + docker build -t $LOCAL_IMAGE_TAG --file $AWS_RUNTIME_DOCKERFILE_PATH --build-arg docker_builder_image=builder .; + fi + + # Push to the local container registry + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker push $LOCAL_IMAGE_TAG + # Push to Amazon's container registry + - $(aws ecr get-login --no-include-email --region $AWS_ECR_REGION) + - docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA + - docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:latest + - docker push $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA + - docker push $AWS_IMAGE_TAG_BASE:latest + only: + variables: + - $AWS == 'true' + changes: + - app/sdms/**/* + + +aws-update-eks: + extends: + - .aws + - .aws_variables + - .aws_common_variables + stage: deploy + needs: ['aws-containerize'] + script: + - export KUBECONFIG=/tmp/kubeconfig-${RANDOM}.yaml + - export EKS_CLUSTER_MGMT_ROLE=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterManagementRole'].OutputValue" --output text) + - export EKS_CLUSTER_NAME=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterName'].OutputValue" --output text) + - echo Using Role -- $EKS_CLUSTER_MGMT_ROLE + - aws eks update-kubeconfig --region $AWS_REGION --name $EKS_CLUSTER_NAME --role-arn $EKS_CLUSTER_MGMT_ROLE + #Some CLIs require a restrictive KUBECONFIG file + - chmod 644 $KUBECONFIG + - kubectl -n osdu-services rollout restart deployment/${AWS_EKS_DEPLOYMENT_NAME} + - kubectl -n osdu-services rollout status -w deployment/${AWS_EKS_DEPLOYMENT_NAME} --timeout=300s + only: + variables: + - $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true' && $AWS_DEPLOY_TARGET == 'EKS' + changes: + - app/sdms/**/* + +aws-update-ecs: + extends: + - .aws + - .aws_common_variables + - .aws_variables + stage: deploy + needs: ['aws-containerize'] + script: + - ECS_SERVICE_NAME=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/ecs/services/${SERVICE_NAME} --query Parameter.Value --output text --region $AWS_REGION) + #limit output to the first 50 lines...the rest is bloat + - export ECS_UPDATE_TMP_FILE=/tmp/ecs-update-log-${RANDOM}.txt + - aws ecs update-service --cluster $AWS_CLUSTER_NAME --service $ECS_SERVICE_NAME --region $AWS_REGION --force-new-deployment > $ECS_UPDATE_TMP_FILE + - cat $ECS_UPDATE_TMP_FILE | head -n 50 + only: + variables: + - $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true' && ($AWS_DEPLOY_TARGET == 'ECS' || $AWS_DEPLOY_TARGET == null) + changes: + - app/sdms/**/* \ No newline at end of file diff --git a/app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml b/app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml new file mode 100644 index 00000000..c64b7ae7 --- /dev/null +++ b/app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml @@ -0,0 +1,24 @@ +push_runtime_image_azure: + image: danielscholl/azure-build-image + tags: ["osdu-medium"] + stage: containerize + variables: + SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} + LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest + before_script: + - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID + - az acr login -n $AZURE_REGISTRY + script: + # Azure Container Registry + - cd app/sdms + - docker build -t builder_sdms:latest --file docker/builder.Dockerfile . + - docker build -t $UTEST_RUNTIME_IMAGE --file docker/runtime.Dockerfile --build-arg docker_builder_image=builder_sdms . + - docker tag $UTEST_RUNTIME_IMAGE ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE + - docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE + - docker tag $UTEST_RUNTIME_IMAGE ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE + - docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE + only: + variables: + - $AZURE == 'true' + changes: + - app/sdms/**/* \ No newline at end of file diff --git a/app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml b/app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml new file mode 100644 index 00000000..6b7d27d3 --- /dev/null +++ b/app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml @@ -0,0 +1,19 @@ +push_runtime_image_gcp: + image: gcr.io/google.com/cloudsdktool/cloud-sdk + tags: ["osdu-medium"] + stage: containerize + variables: + SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} + LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest + before_script: + - gcloud auth activate-service-account --key-file="$GCP_DEPLOY_FILE" + - gcloud config set project $GCP_PROJECT + script: + # GCP Container Registry + - cd app/sdms + - gcloud builds submit --config provider/$GCP_VENDOR/cloudbuild/cloudbuild.yaml --substitutions=_GCP_SERVICE=$GCP_SERVICE,_APPLICATION_NAME=$GCP_APPLICATION_NAME,_PROVIDER_NAME=$GCP_VENDOR,_SHORT_SHA=$CI_COMMIT_SHORT_SHA,_PORT=$PORT + only: + variables: + - $GCP == 'true' + changes: + - app/sdms/**/* \ No newline at end of file diff --git a/app/sdms/devops/osdu/contenerize/seismic-store-service.yml b/app/sdms/devops/osdu/contenerize/seismic-store-service.yml new file mode 100644 index 00000000..2463267f --- /dev/null +++ b/app/sdms/devops/osdu/contenerize/seismic-store-service.yml @@ -0,0 +1,23 @@ +push_runtime_image: + image: docker:19.03.12 + services: + - docker:19.03.12-dind + tags: ["osdu-medium"] + stage: containerize + variables: + SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} + LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + script: + # Gitlab Container Registry + - cd app/sdms + - docker build -t builder_sdms:latest --file docker/builder.Dockerfile . + - docker build -t $UTEST_RUNTIME_IMAGE --file docker/runtime.Dockerfile --build-arg docker_builder_image=builder_sdms . + - docker tag $UTEST_RUNTIME_IMAGE $CI_REGISTRY_IMAGE/$SHA_IMAGE + - docker push $CI_REGISTRY_IMAGE/$SHA_IMAGE + - docker tag $UTEST_RUNTIME_IMAGE $CI_REGISTRY_IMAGE/$LATEST_IMAGE + - docker push $CI_REGISTRY_IMAGE/$LATEST_IMAGE + only: + changes: + - app/sdms/**/* \ No newline at end of file diff --git a/app/sdms/devops/aws/awstest.yml b/app/sdms/devops/osdu/integration/awstest.yml similarity index 100% rename from app/sdms/devops/aws/awstest.yml rename to app/sdms/devops/osdu/integration/awstest.yml diff --git a/app/sdms/devops/aws/requirements.txt b/app/sdms/devops/osdu/integration/requirements.txt similarity index 100% rename from app/sdms/devops/aws/requirements.txt rename to app/sdms/devops/osdu/integration/requirements.txt -- GitLab From 9440de53303d5e7f5e96486f84bcc9e660f5e1c3 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 10:54:07 -0500 Subject: [PATCH 05/45] ci: uodating contenerize and integration steps --- .../osdu/cloud-providers/azure-seismic-store-service.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml index 48d96668..c39908da 100644 --- a/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml @@ -52,6 +52,8 @@ azure_deploy: only: variables: - $AZURE == 'true' + changes: + - app/sdms/**/* azure_test: tags: ["osdu-medium"] @@ -76,3 +78,5 @@ azure_test: only: variables: - $AZURE == 'true' + changes: + - app/sdms/**/* \ No newline at end of file -- GitLab From d9fe87d781028a0c2125bc177405d5b3c2183420 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 10:57:11 -0500 Subject: [PATCH 06/45] ci: updating contenerize and integration steps --- app/sdms/.gitlab-ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml index bc36a50e..5e947872 100644 --- a/app/sdms/.gitlab-ci.yml +++ b/app/sdms/.gitlab-ci.yml @@ -74,7 +74,4 @@ include: # aws - local: "app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml" - # integration - # aws - - local: "app/sdms/devops/osdu/integration/awstest.yml" - + \ No newline at end of file -- GitLab From 3e5b015dc254d141dbadea3e740e59af004db72f Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 10:57:55 -0500 Subject: [PATCH 07/45] ci: updating contenerize and integration steps --- app/sdms/.gitlab-ci.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml index 5e947872..214bba31 100644 --- a/app/sdms/.gitlab-ci.yml +++ b/app/sdms/.gitlab-ci.yml @@ -67,11 +67,4 @@ include: # gcp - project: "app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml" - # deploy - # azure - - local: "app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml" - - # aws - - local: "app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml" - \ No newline at end of file -- GitLab From 0fc5a27e5401bc0788c1ec6c893cf14c520609f7 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 11:15:13 -0500 Subject: [PATCH 08/45] ci: updating contenerize and integration steps --- app/sdms/.gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml index 214bba31..2b26c136 100644 --- a/app/sdms/.gitlab-ci.yml +++ b/app/sdms/.gitlab-ci.yml @@ -67,4 +67,8 @@ include: # gcp - project: "app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml" + # deploy + # azure + - local: "app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml" + \ No newline at end of file -- GitLab From 2f314e997cce79a7d68f5c209d28189b7616b65a Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 11:26:39 -0500 Subject: [PATCH 09/45] ci: updating contenerize and integration steps --- app/sdms/.gitlab-ci.yml | 5 +--- .../azure-seismic-store-service.yml | 26 ++++++++++++++++++- .../azure-seismic-store-service.yml | 24 ----------------- 3 files changed, 26 insertions(+), 29 deletions(-) diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml index 2b26c136..515d876c 100644 --- a/app/sdms/.gitlab-ci.yml +++ b/app/sdms/.gitlab-ci.yml @@ -61,9 +61,6 @@ include: # osdu - project: "app/sdms/devops/osdu/contenerize/seismic-store-service.yml" - # azure - - project: "app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml" - # gcp - project: "app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml" @@ -71,4 +68,4 @@ include: # azure - local: "app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml" - \ No newline at end of file + \ No newline at end of file diff --git a/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml index c39908da..71cab91c 100644 --- a/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml @@ -11,7 +11,31 @@ variables: POD_IDENTITY: osdu-identity REDIS_INSTANCE_PORT: 6380 - +push_runtime_image_azure: + image: danielscholl/azure-build-image + tags: ["osdu-medium"] + stage: containerize + variables: + SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} + LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest + before_script: + - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID + - az acr login -n $AZURE_REGISTRY + script: + # Azure Container Registry + - cd app/sdms + - docker build -t builder_sdms:latest --file docker/builder.Dockerfile . + - docker build -t $UTEST_RUNTIME_IMAGE --file docker/runtime.Dockerfile --build-arg docker_builder_image=builder_sdms . + - docker tag $UTEST_RUNTIME_IMAGE ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE + - docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE + - docker tag $UTEST_RUNTIME_IMAGE ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE + - docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE + only: + variables: + - $AZURE == 'true' + changes: + - app/sdms/**/* + azure_deploy: tags: ["osdu-medium"] image: danielscholl/azure-build-image diff --git a/app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml b/app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml index c64b7ae7..e69de29b 100644 --- a/app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml +++ b/app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml @@ -1,24 +0,0 @@ -push_runtime_image_azure: - image: danielscholl/azure-build-image - tags: ["osdu-medium"] - stage: containerize - variables: - SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} - LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest - before_script: - - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID - - az acr login -n $AZURE_REGISTRY - script: - # Azure Container Registry - - cd app/sdms - - docker build -t builder_sdms:latest --file docker/builder.Dockerfile . - - docker build -t $UTEST_RUNTIME_IMAGE --file docker/runtime.Dockerfile --build-arg docker_builder_image=builder_sdms . - - docker tag $UTEST_RUNTIME_IMAGE ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE - - docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE - - docker tag $UTEST_RUNTIME_IMAGE ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE - - docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE - only: - variables: - - $AZURE == 'true' - changes: - - app/sdms/**/* \ No newline at end of file -- GitLab From b73e8418bb87d8e53fd08d91a5ee7aa8152ce7e8 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 11:27:50 -0500 Subject: [PATCH 10/45] ci: updating azure steps --- app/sdms/.gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml index 515d876c..d5955b31 100644 --- a/app/sdms/.gitlab-ci.yml +++ b/app/sdms/.gitlab-ci.yml @@ -67,5 +67,4 @@ include: # deploy # azure - local: "app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml" - - \ No newline at end of file + \ No newline at end of file -- GitLab From b314bf607c49344dc0f29c4297331bb80c32c1ce Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 12:18:17 -0500 Subject: [PATCH 11/45] ci: updating ci cd pipeline --- app/sdms/.gitlab-ci.yml | 49 +--- .../aws-requirements.txt} | 0 .../aws-seismic-store-service.yml | 38 ++- .../azure-seismic-store-service.yml | 2 + .../gcp-seismic-store-service.yml | 266 ++++++++++++++++++ .../ibm-seismic-store-service.yml | 93 ++++++ .../azure-seismic-store-service.yml | 0 .../contenerize/gcp-seismic-store-service.yml | 19 -- app/sdms/devops/osdu/integration/awstest.yml | 22 -- 9 files changed, 409 insertions(+), 80 deletions(-) rename app/sdms/devops/osdu/{integration/requirements.txt => cloud-providers/aws-requirements.txt} (100%) create mode 100644 app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml create mode 100644 app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml delete mode 100644 app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml delete mode 100644 app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml delete mode 100644 app/sdms/devops/osdu/integration/awstest.yml diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml index d5955b31..c37e34e9 100644 --- a/app/sdms/.gitlab-ci.yml +++ b/app/sdms/.gitlab-ci.yml @@ -8,40 +8,6 @@ variables: SDMS_MIN_REPLICAS: 1 SDMS_MAX_REPLICAS: 5 - #aws variables - AWS_SERVICE: seismic-store - AWS_BUILD_SUBDIR: src/cloud/providers/aws/build-aws - AWS_TEST_SUBDIR: tests - AWS_ENVIRONMENT: dev - AWS_DEPLOY_TARGET: EKS - AWS_EKS_DEPLOYMENT_NAME: os-seismic-store - AWS_BUILDER_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/builder.Dockerfile - AWS_RUNTIME_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/runtime.Dockerfile - # skipping tests here. Using a local file to run tests - AWS_SKIP_TESTS: "true" - #azure variables - AZURE_SERVICE: seismic-store-service - #gcp variables - GCP_APPLICATION_NAME: os-seismic-store-service - GCP_DEPLOY_FILE: $OSDU_GCP_DEPLOY_FILE - GCP_PROJECT: opendes-evt - GCP_SERVICE: seismic-store-service - GCP_VENDOR: gcp - #ibm variables - IBM_OPENSHIFT_PROJECT: og-cicd - IBM_VENDOR: ibm - - OSDU_GCP_LOG_LEVEL: INFO - OSDU_GCP_SERVICE: seismic-store - OSDU_GCP_VENDOR: gcp - OSDU_GCP_APPLICATION_NAME: seismic-store - OSDU_GCP_APPLICATION: seismic-store - OSDU_GCP_ENTITLEMENT_BASE_URL_PATH: /entitlements/v2 - OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY: data-partition-id - OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE: https://community.osdu-gcp.go3-nrg.projects.epam.com/api - OSDU_GCP_DES_SERVICE_HOST_STORAGE: https://os-storage-attcrcktoa-uc.a.run.app/api - OSDU_GCP_ENV_VARS: CLOUDPROVIDER=${OSDU_GCP_CLOUD_PROVIDER},DES_SERVICE_HOST_PARTITION=${OSDU_GCP_PARTITION_API},ENTITLEMENT_BASE_URL_PATH=${OSDU_GCP_ENTITLEMENT_BASE_URL_PATH},DATA_PARTITION_REST_HEADER_KEY=${OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY},DES_SERVICE_HOST_STORAGE=${OSDU_GCP_DES_SERVICE_HOST_STORAGE},DES_SERVICE_HOST_COMPLIANCE=${OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE},SEISTORE_DES_TARGET_AUDIENCE=${GOOGLE_AUDIENCE},SERVICE_CLOUD_PROJECT=${OSDU_GCP_PROJECT},APP_ENVIRONMENT_IDENTIFIER=${TENANT},IMP_SERVICE_ACCOUNT_SIGNER=${OSDU_GCP_IMP_SERVICE_ACCOUNT_SIGNER},DES_SERVICE_HOST_ENTITLEMENT=${OSDU_GCP_ENTITLEMENTS_V2_BASE_URL},SEISTORE_DES_APPKEY=${OSDU_GCP_SEISTORE_DES_APPKEY},DES_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_DES_REDIS_INSTANCE_ADDRESS},DES_REDIS_INSTANCE_PORT=${OSDU_GCP_DES_REDIS_INSTANCE_PORT},LOCKSMAP_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_LOCKSMAP_REDIS_INSTANCE_ADDRESS} --vpc-connector=$OSDU_GCP_VPC_CONNECTOR - include: # build @@ -60,11 +26,18 @@ include: # containerize # osdu - project: "app/sdms/devops/osdu/contenerize/seismic-store-service.yml" - - # gcp - - project: "app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml" # deploy # azure - local: "app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml" - \ No newline at end of file + + # aws + - local: "app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml" + + # gcp + - local: "app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml" + + # ibm + - local: "app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml" + + diff --git a/app/sdms/devops/osdu/integration/requirements.txt b/app/sdms/devops/osdu/cloud-providers/aws-requirements.txt similarity index 100% rename from app/sdms/devops/osdu/integration/requirements.txt rename to app/sdms/devops/osdu/cloud-providers/aws-requirements.txt diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index f5323508..6e8fc6b4 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -1,3 +1,16 @@ +variables: + #aws variables + AWS_SERVICE: seismic-store + AWS_BUILD_SUBDIR: src/cloud/providers/aws/build-aws + AWS_TEST_SUBDIR: tests + AWS_ENVIRONMENT: dev + AWS_DEPLOY_TARGET: EKS + AWS_EKS_DEPLOYMENT_NAME: os-seismic-store + AWS_BUILDER_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/builder.Dockerfile + AWS_RUNTIME_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/runtime.Dockerfile + # skipping tests here. Using a local file to run tests + AWS_SKIP_TESTS: "true" + .aws_variables: before_script: @@ -226,4 +239,27 @@ aws-update-ecs: variables: - $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true' && ($AWS_DEPLOY_TARGET == 'ECS' || $AWS_DEPLOY_TARGET == null) changes: - - app/sdms/**/* \ No newline at end of file + - app/sdms/**/* + +aws-test-newman: + extends: + - .aws + - .aws_common_variables + - .aws_variables + stage: integration + image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-node/aws-node:v1.0-node14 + needs: [{ job: 'aws-update-ecs', optional: true }, { job: 'aws-update-eks', optional: true }] + script: + - pip3 install -r devops/aws/requirements.txt + - svctoken=$(python3 devops/scripts/aws_jwt_client.py) + - npm install -g newman + - chmod +x ./tests/e2e/run_e2e_tests.sh + - ./tests/e2e/run_e2e_tests.sh --seistore-svc-url=${SEISMICSTORE_SVC_URL} --seistore-svc-api-key="NA" --user-idtoken="$svctoken" --user-idtoken="$svctoken" --tenant=opendes --subproject=awsdemosubproject --admin-email="${AWS_COGNITO_AUTH_PARAMS_USER}" --datapartition=opendes --legaltag01=opendes-sdmstestlegaltag --legaltag02=opendes-sdmstestlegaltag --subproject-long-name=subprojectlonggggggggggggggggggggggname --VCS-Provider="${ISGITLAB}" + only: + variables: + - $AWS == 'true' + artifacts: + when: always + paths: + - $INTEGRATION_TEST_DIR + expire_in: 14 days \ No newline at end of file diff --git a/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml index 71cab91c..95eaa53f 100644 --- a/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml @@ -1,4 +1,6 @@ variables: + #azure variables + AZURE_SERVICE: seismic-store-service CHART_PATH: devops/azure/chart CLOUD_PROVIDER: azure E2E_ADMIN_EMAIL: integration.test@azureglobal1.onmicrosoft.com diff --git a/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml new file mode 100644 index 00000000..d0f50a9a --- /dev/null +++ b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml @@ -0,0 +1,266 @@ +variables: + #gcp variables + GCP_APPLICATION_NAME: os-seismic-store-service + GCP_DEPLOY_FILE: $OSDU_GCP_DEPLOY_FILE + GCP_PROJECT: opendes-evt + GCP_SERVICE: seismic-store-service + GCP_VENDOR: gcp + OSDU_GCP_LOG_LEVEL: INFO + OSDU_GCP_SERVICE: seismic-store + OSDU_GCP_VENDOR: gcp + OSDU_GCP_APPLICATION_NAME: seismic-store + OSDU_GCP_APPLICATION: seismic-store + OSDU_GCP_ENTITLEMENT_BASE_URL_PATH: /entitlements/v2 + OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY: data-partition-id + OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE: https://community.osdu-gcp.go3-nrg.projects.epam.com/api + OSDU_GCP_DES_SERVICE_HOST_STORAGE: https://os-storage-attcrcktoa-uc.a.run.app/api + OSDU_GCP_ENV_VARS: CLOUDPROVIDER=${OSDU_GCP_CLOUD_PROVIDER},DES_SERVICE_HOST_PARTITION=${OSDU_GCP_PARTITION_API},ENTITLEMENT_BASE_URL_PATH=${OSDU_GCP_ENTITLEMENT_BASE_URL_PATH},DATA_PARTITION_REST_HEADER_KEY=${OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY},DES_SERVICE_HOST_STORAGE=${OSDU_GCP_DES_SERVICE_HOST_STORAGE},DES_SERVICE_HOST_COMPLIANCE=${OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE},SEISTORE_DES_TARGET_AUDIENCE=${GOOGLE_AUDIENCE},SERVICE_CLOUD_PROJECT=${OSDU_GCP_PROJECT},APP_ENVIRONMENT_IDENTIFIER=${TENANT},IMP_SERVICE_ACCOUNT_SIGNER=${OSDU_GCP_IMP_SERVICE_ACCOUNT_SIGNER},DES_SERVICE_HOST_ENTITLEMENT=${OSDU_GCP_ENTITLEMENTS_V2_BASE_URL},SEISTORE_DES_APPKEY=${OSDU_GCP_SEISTORE_DES_APPKEY},DES_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_DES_REDIS_INSTANCE_ADDRESS},DES_REDIS_INSTANCE_PORT=${OSDU_GCP_DES_REDIS_INSTANCE_PORT},LOCKSMAP_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_LOCKSMAP_REDIS_INSTANCE_ADDRESS} --vpc-connector=$OSDU_GCP_VPC_CONNECTOR + +push_runtime_image_gcp: + image: gcr.io/google.com/cloudsdktool/cloud-sdk + tags: ["osdu-medium"] + stage: containerize + variables: + SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} + LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest + before_script: + - gcloud auth activate-service-account --key-file="$GCP_DEPLOY_FILE" + - gcloud config set project $GCP_PROJECT + script: + # GCP Container Registry + - cd app/sdms + - gcloud builds submit --config provider/$GCP_VENDOR/cloudbuild/cloudbuild.yaml --substitutions=_GCP_SERVICE=$GCP_SERVICE,_APPLICATION_NAME=$GCP_APPLICATION_NAME,_PROVIDER_NAME=$GCP_VENDOR,_SHORT_SHA=$CI_COMMIT_SHORT_SHA,_PORT=$PORT + only: + variables: + - $GCP == 'true' + changes: + - app/sdms/**/* + +# Copyright 2020 Google LLC +# Copyright 2017-2019, Schlumberger +# Copyright 2020 EPAM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Global Protected Variables: +# +# OSDU_GCP +# OSDU_GCP_DEPLOY_FILE +# OSDU_GCP_SERVICE_ACCOUNT +# OSDU_GCP_INTEGRATION_TESTER +# OSDU_GCP_NO_DATA_ACCESS_TESTER + +include: + - local: 'cloud-providers/osdu-gcp-global.yml' + +.osdu-gcp-variables: + variables: + # services' urls + OSDU_GCP_STORAGE_HOSTNAME: community.osdu-gcp.go3-nrg.projects.epam.com + OSDU_GCP_LEGALTAG_API: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/legal/v1 + OSDU_GCP_CRS_API: https://crs-converter-attcrcktoa-uc.a.run.app/api/crs/v2 + OSDU_GCP_REGISTER_URL: https://os-register-attcrcktoa-uc.a.run.app/ + OSDU_GCP_ENTITLEMENTS_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/entitlements/v2/ + OSDU_GCP_ENTITLEMENTS_V2_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/entitlements/v2/ + OSDU_GCP_ENTITLEMENTS_V2_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api + OSDU_GCP_AUTHORIZE_API: https://community.osdu-gcp.go3-nrg.projects.epam.com/entitlements/v2/ + OSDU_GCP_STORAGE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/storage/v2/ + OSDU_GCP_RECORDS_ROOT_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/storage/v2/ + OSDU_GCP_LEGAL_HOST_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/legal/v1/ + OSDU_GCP_LEGAL_HOSTNAME: https://community.osdu-gcp.go3-nrg.projects.epam.com/ + OSDU_GCP_AIRFLOW_URL: https://y38fc16b71c3f21fbp-tp.appspot.com/ + OSDU_GCP_DELIVERY_SERVICE_URL: https://os-delivery-attcrcktoa-uc.a.run.app/api/delivery/v2/ + OSDU_GCP_INGEST_HOST: https://os-ingest-attcrcktoa-uc.a.run.app/ + OSDU_GCP_WORKFLOW_SERVICE_URL: https://os-workflow-attcrcktoa-uc.a.run.app/api/workflow/ + OSDU_GCP_SEARCH_HOST: https://os-search-attcrcktoa-uc.a.run.app/api/search/v2/ + OSDU_GCP_BACKUP_HOST: https://backup-attcrcktoa-uc.a.run.app + OSDU_GCP_SEARCH_QUERY_URL: https://os-search-attcrcktoa-uc.a.run.app/api/search/v2/query/ + OSDU_GCP_INDEXER_HOST: https://community.osdu-gcp.go3-nrg.projects.epam.com/ + OSDU_GCP_INDEXER_HOST_SEARCH: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/indexer/v2/ + OSDU_GCP_FILE_URL: https://os-file-attcrcktoa-uc.a.run.app + OSDU_GCP_NOTIFICATION_URL: https://os-notification-attcrcktoa-uc.a.run.app/api/notification/v1/ + OSDU_GCP_STORAGE_QUERY_RECORD_HOST: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/storage/v2/query/records + OSDU_GCP_STORAGE_SCHEMA_HOST: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/storage/v2/schemas + OSDU_GCP_STORAGE_QUERY_RECORD_FOR_CONVERSION_HOST: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/storage/v2/query/records:batch + OSDU_GCP_INDEXER_QUEUE_HOST: https://indexer-queue-attcrcktoa-uc.a.run.app/api/indexer/v1/_dps/task-handlers/enqueue + OSDU_GCP_UNIT_HOSTNAME: os-unit-attcrcktoa-uc.a.run.app + OSDU_GCP_CRS_CONVERTER_HOSTNAME: crs-converter-attcrcktoa-uc.a.run.app + OSDU_GCP_CRS_CATALOG_HOSTNAME: os-crs-catalog-attcrcktoa-uc.a.run.app + OSDU_GCP_SCHEMA_URL: https://os-schema-attcrcktoa-uc.a.run.app + OSDU_GCP_DATASET_URL: https://os-dataset-attcrcktoa-uc.a.run.app/api/dataset/v1/ + OSDU_GCP_PARTITION_API: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/partition/v1/ + OSDU_GCP_POLICY_API: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/policy/v1/ + OSDU_GCP_LOCAL_IMAGE_TAG_SHA: $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA + REDIS_GROUP_HOST: 10.234.198.27 + REDIS_SEARCH_HOST: 10.99.138.107 + REDIS_STORAGE_HOST: 10.16.59.203 + REDIS_SYNC_HOST: 10.116.62.35 + OSDU_GCP_REDIS_SEARCH_PORT: 6379 + OSDU_GCP_SPRING_PROFILES_ACTIVE: dev + OSDU_GCP_INDEXER_QUEUE_CLOUDTASK_NAME: os-indexer-queue-osdu + # variables for unit deployment + OSDU_GCP_UNIT_CATALOG_BUCKET: nice-etching-277309-unit-catalog-bucket + # common variables + OSDU_GCP_CLOUDRUN_REGION: us-central1 + OSDU_GCP_PROJECT: nice-etching-277309 + OSDU_GCP_VPC_CONNECTOR: osdu-connector + OSDU_GCP_PORT: 8080 + #variables for seismic-store deployment + OSDU_GCP_CLOUD_PROVIDER: google + OSDU_GCP_IMP_SERVICE_ACCOUNT_SIGNER: unknown + OSDU_GCP_SEISTORE_DES_APPKEY: na + OSDU_GCP_DES_REDIS_INSTANCE_ADDRESS: 10.93.210.3 + OSDU_GCP_DES_REDIS_INSTANCE_PORT: 6379 + OSDU_GCP_LOCKSMAP_REDIS_INSTANCE_ADDRESS: 10.93.210.3 + # variables for WKS service: + OSDU_GCP_WKS_SEARCH_API: https://os-search-attcrcktoa-uc.a.run.app/api/search/v2 + OSDU_GCP_WKS_STORAGE_API: https://community.osdu-gcp.go3-nrg.projects.epam.com/api/storage/v2 + OSDU_GCP_WKS_SCHEMA_API: https://os-schema-attcrcktoa-uc.a.run.app/api/schema-service/v1 + # common variables for tests + GOOGLE_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com + DOMAIN: osdu-gcp.go3-nrg.projects.epam.com + LEGAL_TAG: osdu-demo-legaltag + OTHER_RELEVANT_DATA_COUNTRIES: US + DEFAULT_DATA_PARTITION_ID_TENANT1: osdu + GCLOUD_PROJECT: nice-etching-277309 + # variables for storage tests + TENANT_NAME: osdu + # variables for workflow tests + OSDU_GCP_ENTITLEMENTS_APPKEY: workflow-service + FINISHED_WORKFLOW_ID: fad778da-fbc4-4261-8b3e-deb48be44969 + OSDU_GCP_TEST_DAG_NAME: airflow_monitoring + # variables for ingestion tests + TEST_OSDU_FILE_PATH: gs://nice-etching-277309-file/r1/data/provided/well-logs/1013_akm11_1978_comp.las + # variables for file deployment and tests + OSDU_GCP_DATASTORE_NAMESPACE: osdu-namespace + FILE_BUCKET: nice-etching-277309-file + ACL_OWNERS: data.default.owners + ACL_VIEWERS: data.default.viewers + TARGET_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com + DATA_PARTITION_ID: osdu + USER_ID: common-user + TIME_ZONE: UTC+0 + # variables for delivery tests + ENTITLEMENTS_DOMAIN: osdu-gcp.go3-nrg.projects.epam.com + TENANT: osdu + #DEFAULT_DATA_PARTITION_ID_TENANT2: osdu + INTEGRATION_TEST_AUDIENCE: 689762842995-pv217jo3k8j803kk6gqf52qb5amos3a9.apps.googleusercontent.com + # variables for register tests, and deployment + SERVICE_IDENTITY: osdu-gcp-sa + RECORDS_CHANGE_PUBSUB_ENDPOINT: https://os-notification-attcrcktoa-uc.a.run.app/api/notification/v1/push-handlers/records-changed + CRON_JOB_EXPECTED_IP: 0:0:0:0:0:0:0:1 + SUBSCRIBER_PRIVATE_KEY_ID: $OSDU_GCP_SUBSCRIBER_PRIVATE_KEY_ID + SUBSCRIBER_SECRET: $OSDU_GCP_SUBSCRIBER_SECRET + SUBSCRIPTION_ID: $OSDU_GCP_REGISTER_SUBSCRIPTION_ID + REGISTER_CUSTOM_PUSH_URL: https://os-register-attcrcktoa-uc.a.run.app/api/register/v1/test/challenge/1 + CLIENT_TENANT: nonexistenttenant + OSDU_TENANT: osdu + REGISTER_BASE_URL: https://os-register-attcrcktoa-uc.a.run.app/ + ENVIRONMENT: dev + # variables for notification tests + HMAC_SECRET: $OSDU_GCP_SUBSCRIBER_SECRET + REGISTER_CUSTOM_PUSH_URL_HMAC: https://os-register-attcrcktoa-uc.a.run.app/api/register/v1/test/challenge/hmac-integration-test + TOPIC_ID: records-changed + NOTIFICATION_BASE_URL: $OSDU_GCP_NOTIFICATION_URL + # variables for legal tests + MY_TENANT_PROJECT: osdu + MY_TENANT: osdu + ENABLE_FULL_BUCKET_NAME: "true" + SKIP_HTTP_TESTS: "true" + # variables for search tests + DATA_GROUP: osdu + ENTITLEMENTS_HOST: $OSDU_GCP_ENTITLEMENTS_URL + ELASTIC_PASSWORD: $OSDU_GCP_ELASTIC_PASSWORD + ELASTIC_USER_NAME: $OSDU_GCP_ELASTIC_USER + ELASTIC_HOST: $OSDU_GCP_ELASTIC_HOST + ELASTIC_PORT: $OSDU_GCP_ELASTIC_PORT + DATA_PARTITION_ID_TENANT1: osdu + # variables for schema tests + VENDOR: $OSDU_GCP_VENDOR + HOST: $OSDU_GCP_SCHEMA_URL + PRIVATE_TENANT1: osdu + PRIVATE_TENANT2: osdu + SHARED_TENANT: osdu + # variables for partition tests + PARTITION_BASE_URL: https://community.osdu-gcp.go3-nrg.projects.epam.com/ + # variables for backup tests + BACKUP_SERVICE_HOST: $OSDU_GCP_BACKUP_HOST + +osdu-gcp-containerize-gcloud: + stage: containerize + needs: ["compile-and-unit-test"] + extends: .osdu-gcp-variables + image: gcr.io/google.com/cloudsdktool/cloud-sdk + cache: {} + script: + - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE + - gcloud config set project $OSDU_GCP_PROJECT + - touch .gcloudignore + - gcloud builds submit --config provider/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR/cloudbuild/cloudbuild.yaml --substitutions=_GCP_SERVICE=$OSDU_GCP_SERVICE,_APPLICATION_NAME=$OSDU_GCP_APPLICATION_NAME,_PROVIDER_NAME=$OSDU_GCP_VENDOR,_SHORT_SHA=$CI_COMMIT_SHORT_SHA,_PORT=$OSDU_GCP_PORT + only: + variables: + - $OSDU_GCP == 'true' + +osdu-gcp-containerize-gitlab: + stage: containerize + needs: ["compile-and-unit-test"] + tags: ["osdu-medium"] + extends: .osdu-gcp-variables + image: docker:19.03 + cache: {} + allow_failure: true + script: + - export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:latest" ; fi + - docker build -t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --file docker/runtime.Dockerfile . + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker push $CI_REGISTRY_IMAGE/osdu-gcp + only: + variables: + - $OSDU_GCP == 'true' + +osdu-gcp-deploy: + image: gcr.io/google.com/cloudsdktool/cloud-sdk + needs: ["osdu-gcp-containerize-gcloud"] + stage: deploy + extends: .osdu-gcp-variables + cache: {} + script: + - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE + - gcloud config set project $OSDU_GCP_PROJECT + - > + gcloud beta run deploy $OSDU_GCP_APPLICATION_NAME + --image gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA + --platform managed + --region $OSDU_GCP_CLOUDRUN_REGION + --allow-unauthenticated + --service-account $OSDU_GCP_SERVICE_ACCOUNT + --memory 512M + $OSDU_GCP_CLOUD_RUN_PARAMETERS + --set-env-vars=LOG_LEVEL=${OSDU_GCP_LOG_LEVEL:="INFO"},$OSDU_GCP_ENV_VARS + - gcloud run services update-traffic $OSDU_GCP_APPLICATION_NAME --platform managed --region $OSDU_GCP_CLOUDRUN_REGION --to-latest + only: + variables: + - $OSDU_GCP == 'true' + +osdu-gcp-tagging: + image: google/cloud-sdk + needs: ["osdu-gcp-deploy"] + stage: integration + extends: .osdu-gcp-variables + cache: {} + script: + - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE + - gcloud config set project $OSDU_GCP_PROJECT + - gcloud container images add-tag gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_TAG + only: + - tags diff --git a/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml new file mode 100644 index 00000000..2e89a982 --- /dev/null +++ b/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml @@ -0,0 +1,93 @@ +variables: + #ibm variables + IBM_OPENSHIFT_PROJECT: og-cicd + IBM_VENDOR: ibm + +.ibm: + tags: ['osdu-medium'] + image: openshift/origin-cli + environment: + name: IBM + + variables: + IBM_OPENSHIFT_PROJECT: og-cicd + IBM_OPENSHIFT_DEVPRI_PROJECT: og-cicd + + before_script: +# - mkdir -p "$IBM_DEPLOY_DIR" + - oc login --token=$IBM_OPENSHIFT_TOKEN --server=$IBM_OPENSHIFT_URL + +ibm-deploy: + extends: .ibm + stage: deploy + needs: ['compile-and-unit-test'] + script: + - echo "In IBM deploy" + - oc project $IBM_OPENSHIFT_PROJECT + - oc get bc/$CI_PROJECT_NAME 2> /dev/null || oc new-build --name $CI_PROJECT_NAME --binary --strategy source --image-stream openshift/ubi8-nodejs-14 + - oc start-build $CI_PROJECT_NAME --from-dir=. --follow + - oc get service $CI_PROJECT_NAME 2> /dev/null || oc new-app $CI_PROJECT_NAME + only: + variables: + - $IBM == 'true' + +ibm-deploy-devpri: + extends: .ibm + stage: deploy + needs: ['compile-and-unit-test'] + script: + - echo "In IBM deploy" + - oc login --token=$IBM_PRIMARY_DEV_TOKEN --server=$IBM_PRIMARY_DEV_URL + - oc project $IBM_OPENSHIFT_DEVPRI_PROJECT + - oc get bc/$CI_PROJECT_NAME 2> /dev/null || oc new-build --name $CI_PROJECT_NAME --binary --strategy source --image-stream openshift/ubi8-nodejs-14 + - oc start-build $CI_PROJECT_NAME --from-dir=. --follow + - oc get service $CI_PROJECT_NAME 2> /dev/null || oc new-app $CI_PROJECT_NAME + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + +.ibm_variables: + + variables: + # Services + SEISMIC_HOST: $IBM_SEISMIC_HOST + + # common security + KEYCLOAK_URL: $IBM_KEYCLOAK_URL + KEYCLOAK_REALM: OSDU + KEYCLOAK_CLIENT_ID: $IBM_KEYCLOAK_CLIENT_ID + KEYCLOAK_CLIENT_SECRET: $IBM_KEYCLOAK_CLIENT_SECRET + + AUTH_USER_ACCESS: $IBM_AUTH_USER_ACCESS + AUTH_USER_ACCESS_PASSWORD: $IBM_AUTH_USER_ACCESS_PASSWORD +ibm-test: + tags: ["osdu-medium"] + image: node + stage: integration + extends: + - .ibm_variables + allow_failure: false + needs: ["ibm-deploy"] + script: + - apt-get update + - apt-get install -y python + - apt-get install -y python-pip + - pip install requests + - svctoken=$(python devops/scripts/ibm_jwt_client.py) + - echo $svctoken + - npm install -g newman + - chmod +x ./tests/e2e/run_e2e_tests.sh + - ./tests/e2e/run_e2e_tests.sh --seistore-svc-url="https://${SEISMIC_HOST}/api/v3" --seistore-svc-api-key="NA" --user-idtoken="$svctoken" --user1-idtoken="$svctoken" --tenant="${IBM_SEISMIC_E2E_TENANT}" --subproject="${IBM_SEISMIC_E2E_SUBPROJECT}" --admin-email="${IBM_SEISMIC_E2E_ADMIN_EMAIL}" --datapartition="${IBM_SEISMIC_E2E_DATA_PARTITION}" --legaltag01="${IBM_SEISMIC_E2E_LEGALTAG01}" --legaltag02="${IBM_SEISMIC_E2E_LEGALTAG02}" --subproject-long-name="${IBM_SEISMIC_E2E_SUBPROJECT_LONG_NAME}" --newuser="${IBM_SEISMIC_E2E_NEWUSER}" --VCS-Provider="gitlab" + only: + variables: + - $IBM == 'true' + +ibm-preship-promote: + extends: .ibm + stage: publish + needs: ['ibm-test'] + script: + - oc login --token=$IBM_PRESHIP_OPENSHIFT_TOKEN --server=$IBM_PRESHIP_OPENSHIFT_URL + - oc project $IBM_OPENSHIFT_PRESHIP_PROJECT + - oc start-build og-$CI_PROJECT_NAME-ibm + rules: + - if: '$CI_COMMIT_BRANCH == "master"' diff --git a/app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml b/app/sdms/devops/osdu/contenerize/azure-seismic-store-service.yml deleted file mode 100644 index e69de29b..00000000 diff --git a/app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml b/app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml deleted file mode 100644 index 6b7d27d3..00000000 --- a/app/sdms/devops/osdu/contenerize/gcp-seismic-store-service.yml +++ /dev/null @@ -1,19 +0,0 @@ -push_runtime_image_gcp: - image: gcr.io/google.com/cloudsdktool/cloud-sdk - tags: ["osdu-medium"] - stage: containerize - variables: - SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} - LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest - before_script: - - gcloud auth activate-service-account --key-file="$GCP_DEPLOY_FILE" - - gcloud config set project $GCP_PROJECT - script: - # GCP Container Registry - - cd app/sdms - - gcloud builds submit --config provider/$GCP_VENDOR/cloudbuild/cloudbuild.yaml --substitutions=_GCP_SERVICE=$GCP_SERVICE,_APPLICATION_NAME=$GCP_APPLICATION_NAME,_PROVIDER_NAME=$GCP_VENDOR,_SHORT_SHA=$CI_COMMIT_SHORT_SHA,_PORT=$PORT - only: - variables: - - $GCP == 'true' - changes: - - app/sdms/**/* \ No newline at end of file diff --git a/app/sdms/devops/osdu/integration/awstest.yml b/app/sdms/devops/osdu/integration/awstest.yml deleted file mode 100644 index 3783a928..00000000 --- a/app/sdms/devops/osdu/integration/awstest.yml +++ /dev/null @@ -1,22 +0,0 @@ -aws-test-newman: - extends: - - .aws - - .aws_common_variables - - .aws_variables - stage: integration - image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-node/aws-node:v1.0-node14 - needs: [{ job: 'aws-update-ecs', optional: true }, { job: 'aws-update-eks', optional: true }] - script: - - pip3 install -r devops/aws/requirements.txt - - svctoken=$(python3 devops/scripts/aws_jwt_client.py) - - npm install -g newman - - chmod +x ./tests/e2e/run_e2e_tests.sh - - ./tests/e2e/run_e2e_tests.sh --seistore-svc-url=${SEISMICSTORE_SVC_URL} --seistore-svc-api-key="NA" --user-idtoken="$svctoken" --user-idtoken="$svctoken" --tenant=opendes --subproject=awsdemosubproject --admin-email="${AWS_COGNITO_AUTH_PARAMS_USER}" --datapartition=opendes --legaltag01=opendes-sdmstestlegaltag --legaltag02=opendes-sdmstestlegaltag --subproject-long-name=subprojectlonggggggggggggggggggggggname --VCS-Provider="${ISGITLAB}" - only: - variables: - - $AWS == 'true' - artifacts: - when: always - paths: - - $INTEGRATION_TEST_DIR - expire_in: 14 days -- GitLab From 46b4722a4234589ddaa27d057d1ee1284a6333a1 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 12:21:55 -0500 Subject: [PATCH 12/45] ci: updating ci cd pipeline --- app/sdms/.gitlab-ci.yml | 4 +--- .../devops/osdu/cloud-providers/gcp-seismic-store-service.yml | 3 ++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml index c37e34e9..9fc43c53 100644 --- a/app/sdms/.gitlab-ci.yml +++ b/app/sdms/.gitlab-ci.yml @@ -38,6 +38,4 @@ include: - local: "app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml" # ibm - - local: "app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml" - - + - local: "app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml" \ No newline at end of file diff --git a/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml index d0f50a9a..6c094817 100644 --- a/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml @@ -61,7 +61,8 @@ push_runtime_image_gcp: # OSDU_GCP_NO_DATA_ACCESS_TESTER include: - - local: 'cloud-providers/osdu-gcp-global.yml' + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/osdu-gcp-global.yml" .osdu-gcp-variables: variables: -- GitLab From 71dcd0717d8ce55b9e93858420006a9e169b041a Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 12:34:08 -0500 Subject: [PATCH 13/45] ci: updating ci cd pipeline --- .../osdu/cloud-providers/aws-seismic-store-service.yml | 2 ++ .../osdu/cloud-providers/gcp-seismic-store-service.yml | 8 ++++++++ .../osdu/cloud-providers/ibm-seismic-store-service.yml | 6 ++++++ 3 files changed, 16 insertions(+) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 6e8fc6b4..9bded999 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -258,6 +258,8 @@ aws-test-newman: only: variables: - $AWS == 'true' + changes: + - app/sdms/**/* artifacts: when: always paths: diff --git a/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml index 6c094817..e74da143 100644 --- a/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml @@ -211,6 +211,8 @@ osdu-gcp-containerize-gcloud: only: variables: - $OSDU_GCP == 'true' + changes: + - app/sdms/**/* osdu-gcp-containerize-gitlab: stage: containerize @@ -228,6 +230,8 @@ osdu-gcp-containerize-gitlab: only: variables: - $OSDU_GCP == 'true' + changes: + - app/sdms/**/* osdu-gcp-deploy: image: gcr.io/google.com/cloudsdktool/cloud-sdk @@ -252,6 +256,8 @@ osdu-gcp-deploy: only: variables: - $OSDU_GCP == 'true' + changes: + - app/sdms/**/* osdu-gcp-tagging: image: google/cloud-sdk @@ -265,3 +271,5 @@ osdu-gcp-tagging: - gcloud container images add-tag gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_TAG only: - tags + changes: + - app/sdms/**/* diff --git a/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml index 2e89a982..9bf6f04c 100644 --- a/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml @@ -30,6 +30,8 @@ ibm-deploy: only: variables: - $IBM == 'true' + changes: + - app/sdms/**/* ibm-deploy-devpri: extends: .ibm @@ -80,6 +82,8 @@ ibm-test: only: variables: - $IBM == 'true' + changes: + - app/sdms/**/* ibm-preship-promote: extends: .ibm @@ -91,3 +95,5 @@ ibm-preship-promote: - oc start-build og-$CI_PROJECT_NAME-ibm rules: - if: '$CI_COMMIT_BRANCH == "master"' + changes: + - app/sdms/**/* -- GitLab From 115c228f8cf0235ffe0be666388d15f20f637f91 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 12:42:41 -0500 Subject: [PATCH 14/45] ci: updating ci cd pipeline --- .../devops/osdu/cloud-providers/gcp-seismic-store-service.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml index e74da143..8e25a7ab 100644 --- a/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml @@ -271,5 +271,3 @@ osdu-gcp-tagging: - gcloud container images add-tag gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_TAG only: - tags - changes: - - app/sdms/**/* -- GitLab From 2a56d54ea9f079aecbde28a992fdc4f05d749055 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 12:44:26 -0500 Subject: [PATCH 15/45] ci: updating ci cd pipeline --- .../osdu/cloud-providers/ibm-seismic-store-service.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml index 9bf6f04c..5e234a3f 100644 --- a/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml @@ -46,6 +46,8 @@ ibm-deploy-devpri: - oc get service $CI_PROJECT_NAME 2> /dev/null || oc new-app $CI_PROJECT_NAME rules: - if: '$CI_COMMIT_BRANCH == "master"' + changes: + - app/sdms/**/* .ibm_variables: @@ -95,5 +97,5 @@ ibm-preship-promote: - oc start-build og-$CI_PROJECT_NAME-ibm rules: - if: '$CI_COMMIT_BRANCH == "master"' - changes: - - app/sdms/**/* + changes: + - app/sdms/**/* -- GitLab From 6e970cb0b4adcaf6bae366c4b7b86a67d39031b2 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 13:10:56 -0500 Subject: [PATCH 16/45] ci: updating readme in root --- README.md | 44 -------------------------------------------- 1 file changed, 44 deletions(-) diff --git a/README.md b/README.md index c5ef000d..bd9bf7dd 100644 --- a/README.md +++ b/README.md @@ -8,47 +8,3 @@ Seismic Store is a cloud-based solution composed by restful micro-services, clie Seismic Store manages data authorization at service level by protecting access to storage bucket resources. Only service authorized users are enabled to directly access a storage resource. The service implements a mechanism that generates an “impersonation token” by authorizing long running/background production jobs to access data without requiring further user interactions. -![service architecture diagram](docs/sdms-architecture-diagram-plain.png "Service Architecture Diagram") - -```python -# build the service -npm run build - -# start the service -npm run start - -# run integral/unit tests -npm run test - -# run integral/unit tests and generate the code coverage -npm run code-coverage - -# run the regression/e2e test suite -./tests/e2e/run_e2e_tests.sh \ - --seistore-svc-url="seismic store service url" \ - --seistore-svc-api-key="seismic store service api key" \ - --user-idtoken="user or service agent idtoken" \ - --tenant="seistore working tenant" \ - --subproject="seistore working subproject" \ - --admin-email="admin email" - -# run the parallel regression/e2e test suite (add the --run-parallel option) -./tests/e2e/run_e2e_tests.sh \ - --seistore-svc-url="seismic store service url" \ - --seistore-svc-api-key="seismic store service api key" \ - --user-idtoken="user or service agent idtoken" \ - --tenant="seistore working tenant" \ - --subproject="seistore working subproject" \ - --admin-email="admin email" \ - --run-parallel - -# run the linter on sources -tslint -c tslint.json -p tsconfig.json 'src/**/*.ts' -``` - -## Environment configuration - -Environment variables can be provided with a `.env` file in the root of the project to be consumed -by [dotenv](https://github.com/motdotla/dotenv). Environment variables are [preloaded](https://github.com/motdotla/dotenv#preload) -by the `npm start` command with the argument `-r dotenv/config`. A template `.env` file can be found -in `/docs/templates/.env-sample`. -- GitLab From 5eecf37dc11d67187f29675b8ac5ad63016e9d2d Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 13:14:49 -0500 Subject: [PATCH 17/45] ci: updating fossa scan --- .../devops/osdu/scanners/fossa-global.yml | 136 ++++++++++++++++++ app/sdms/devops/osdu/scanners/fossa-node.yml | 3 +- 2 files changed, 137 insertions(+), 2 deletions(-) create mode 100644 app/sdms/devops/osdu/scanners/fossa-global.yml diff --git a/app/sdms/devops/osdu/scanners/fossa-global.yml b/app/sdms/devops/osdu/scanners/fossa-global.yml new file mode 100644 index 00000000..36f74cbb --- /dev/null +++ b/app/sdms/devops/osdu/scanners/fossa-global.yml @@ -0,0 +1,136 @@ + +# This checks to see if the NOTICE file generated based on the current code +# matches the one that is committed in the repository. Because FOSSA can +# occassionally order the lines different between executions, we use a python +# program to parse the file into sections and sort each section. Then we compare +# and expect the same number of sections, names of sections, and contents +# (without consider order). +fossa-check-notice: + image: $CI_REGISTRY/divido/fossa-cli/fossa-cli:v6.1 + stage: attribution + needs: ['fossa-analyze'] + tags: ['osdu-small'] + only: + variables: + - $FOSSA_API_KEY + changes: + - app/sdms/**/* + + artifacts: + when: on_failure + paths: + - public + + script: + # fossa-check-for-licensing-issues needs a CI_COMMIT_BRANCH defined to know how to parse the FOSSA API results + # When building tags, this isn't defined by GitLab. In that case, we use the tag name instead. If that's not defined + # then things will fail and we'll have to make this smarter + - test -z "$CI_COMMIT_BRANCH" && export CI_COMMIT_BRANCH="$CI_COMMIT_TAG" + + - mkdir -p public + - fossa report licenses --project "${CI_PROJECT_TITLE}" --branch "${CI_COMMIT_BRANCH}" > public/NOTICE + - mkdir committed-NOTICE generated-NOTICE + - | + python < Date: Tue, 31 Aug 2021 13:25:10 -0500 Subject: [PATCH 18/45] ci: updating requirements --- .../cloud-providers/aws-requirements.txt => aws/requirements.txt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename app/sdms/devops/{osdu/cloud-providers/aws-requirements.txt => aws/requirements.txt} (100%) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-requirements.txt b/app/sdms/devops/aws/requirements.txt similarity index 100% rename from app/sdms/devops/osdu/cloud-providers/aws-requirements.txt rename to app/sdms/devops/aws/requirements.txt -- GitLab From 055474b89dd04e2177a3f3c4f9ac5868542a1faf Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 13:45:02 -0500 Subject: [PATCH 19/45] ci: fixing paths --- .../devops/osdu/cloud-providers/aws-seismic-store-service.yml | 4 ++-- .../osdu/cloud-providers/azure-seismic-store-service.yml | 1 + .../devops/osdu/cloud-providers/gcp-seismic-store-service.yml | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 9bded999..7eaf0f2d 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -172,7 +172,8 @@ aws-containerize: - .aws_variables stage: containerize needs: ['compile-and-unit-test'] - script: + script: + - cd app/sdms - | if [ -z $AWS_BUILDER_DOCKERFILE_PATH ] && [ -z $AWS_RUNTIME_DOCKERFILE_PATH ]; then echo Building BUILD_DIR/Dockerfile container @@ -182,7 +183,6 @@ aws-containerize: docker build -t builder:latest --file $AWS_BUILDER_DOCKERFILE_PATH .; docker build -t $LOCAL_IMAGE_TAG --file $AWS_RUNTIME_DOCKERFILE_PATH --build-arg docker_builder_image=builder .; fi - # Push to the local container registry - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - docker push $LOCAL_IMAGE_TAG diff --git a/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml index 95eaa53f..ea0fa061 100644 --- a/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml @@ -54,6 +54,7 @@ azure_deploy: - az aks get-credentials -g $AZURE_UNIQUE-rg -n $AZURE_UNIQUE-aks script: # Replace values in config file + - cd app/sdms - cp ${CHART_PATH}/helm-config.yaml ${CHART_PATH}/values.yaml - sed -i 's,#{CONTAINER_REGISTRY_NAME}#,'$IMAGE',' ${CHART_PATH}/values.yaml - sed -i 's,#{DNS_HOST}#,'$DES_URL',' ${CHART_PATH}/values.yaml diff --git a/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml index 8e25a7ab..a9a84818 100644 --- a/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml @@ -204,6 +204,7 @@ osdu-gcp-containerize-gcloud: image: gcr.io/google.com/cloudsdktool/cloud-sdk cache: {} script: + - cd app/sdms - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE - gcloud config set project $OSDU_GCP_PROJECT - touch .gcloudignore @@ -223,6 +224,7 @@ osdu-gcp-containerize-gitlab: cache: {} allow_failure: true script: + - cd app/sdms - export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:latest" ; fi - docker build -t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --file docker/runtime.Dockerfile . - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY -- GitLab From 3342882af60eea2f9adb8773ae7d758fb5b959f2 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 14:25:00 -0500 Subject: [PATCH 20/45] ci: fixing paths --- .../devops/osdu/cloud-providers/aws-seismic-store-service.yml | 1 + .../devops/osdu/cloud-providers/azure-seismic-store-service.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 7eaf0f2d..066daf08 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -250,6 +250,7 @@ aws-test-newman: image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-node/aws-node:v1.0-node14 needs: [{ job: 'aws-update-ecs', optional: true }, { job: 'aws-update-eks', optional: true }] script: + - cd app/sdms - pip3 install -r devops/aws/requirements.txt - svctoken=$(python3 devops/scripts/aws_jwt_client.py) - npm install -g newman diff --git a/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml index ea0fa061..dc9d1df6 100644 --- a/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml @@ -93,6 +93,7 @@ azure_test: AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID script: + - cd app/sdms - apt-get update - apt-get install -y python - apt-get install -y python-pip -- GitLab From 6038eac36ede4520ff93f790c6401b0c54931db6 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 14:31:14 -0500 Subject: [PATCH 21/45] ci: fixing paths --- .../devops/osdu/cloud-providers/ibm-seismic-store-service.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml index 5e234a3f..a6c64823 100644 --- a/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml @@ -22,6 +22,7 @@ ibm-deploy: stage: deploy needs: ['compile-and-unit-test'] script: + - cd app/sdms - echo "In IBM deploy" - oc project $IBM_OPENSHIFT_PROJECT - oc get bc/$CI_PROJECT_NAME 2> /dev/null || oc new-build --name $CI_PROJECT_NAME --binary --strategy source --image-stream openshift/ubi8-nodejs-14 @@ -38,6 +39,7 @@ ibm-deploy-devpri: stage: deploy needs: ['compile-and-unit-test'] script: + - cd app/sdms - echo "In IBM deploy" - oc login --token=$IBM_PRIMARY_DEV_TOKEN --server=$IBM_PRIMARY_DEV_URL - oc project $IBM_OPENSHIFT_DEVPRI_PROJECT @@ -72,6 +74,7 @@ ibm-test: allow_failure: false needs: ["ibm-deploy"] script: + - cd app/sdms - apt-get update - apt-get install -y python - apt-get install -y python-pip @@ -92,6 +95,7 @@ ibm-preship-promote: stage: publish needs: ['ibm-test'] script: + - cd app/sdms - oc login --token=$IBM_PRESHIP_OPENSHIFT_TOKEN --server=$IBM_PRESHIP_OPENSHIFT_URL - oc project $IBM_OPENSHIFT_PRESHIP_PROJECT - oc start-build og-$CI_PROJECT_NAME-ibm -- GitLab From a27f6603a499a8d7fd25c6e301ba0ddb30c39864 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 14:48:33 -0500 Subject: [PATCH 22/45] ci: fixing paths --- app/sdms/devops/osdu/scanners/fossa-global.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/app/sdms/devops/osdu/scanners/fossa-global.yml b/app/sdms/devops/osdu/scanners/fossa-global.yml index 36f74cbb..fc92179c 100644 --- a/app/sdms/devops/osdu/scanners/fossa-global.yml +++ b/app/sdms/devops/osdu/scanners/fossa-global.yml @@ -25,6 +25,7 @@ fossa-check-notice: # fossa-check-for-licensing-issues needs a CI_COMMIT_BRANCH defined to know how to parse the FOSSA API results # When building tags, this isn't defined by GitLab. In that case, we use the tag name instead. If that's not defined # then things will fail and we'll have to make this smarter + - cd app/sdms - test -z "$CI_COMMIT_BRANCH" && export CI_COMMIT_BRANCH="$CI_COMMIT_TAG" - mkdir -p public -- GitLab From cfac9ab82e38f7b395c67bb76c2087d4c1ea545c Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 15:20:38 -0500 Subject: [PATCH 23/45] ci: updating readme --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index bd9bf7dd..71d4aecb 100644 --- a/README.md +++ b/README.md @@ -7,4 +7,3 @@ Saving a dataset on a cloud-based storage, as single entity, may be a problem wh Seismic Store is a cloud-based solution composed by restful micro-services, client APIs and tools designed to implement a multi-object storage approach. The system saves objects that compose a dataset as a hierarchical data structure in a cloud storage and the dataset properties as a metadata entry in a no-relational catalogue. Having the datasets stored as multiple independent objects improve the overall performance, as generic I/O operations, for example read or write objects, can be easily parallelized. Seismic Store manages data authorization at service level by protecting access to storage bucket resources. Only service authorized users are enabled to directly access a storage resource. The service implements a mechanism that generates an “impersonation token” by authorizing long running/background production jobs to access data without requiring further user interactions. - -- GitLab From de2b358d0675a29367f68a1fda121242dec8c5d0 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 15:45:01 -0500 Subject: [PATCH 24/45] ci: updating readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 71d4aecb..58f27cac 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,4 @@ Saving a dataset on a cloud-based storage, as single entity, may be a problem wh Seismic Store is a cloud-based solution composed by restful micro-services, client APIs and tools designed to implement a multi-object storage approach. The system saves objects that compose a dataset as a hierarchical data structure in a cloud storage and the dataset properties as a metadata entry in a no-relational catalogue. Having the datasets stored as multiple independent objects improve the overall performance, as generic I/O operations, for example read or write objects, can be easily parallelized. -Seismic Store manages data authorization at service level by protecting access to storage bucket resources. Only service authorized users are enabled to directly access a storage resource. The service implements a mechanism that generates an “impersonation token” by authorizing long running/background production jobs to access data without requiring further user interactions. +Seismic Store manages data authorization at service level by protecting access to storage bucket resources. Only service authorized users are enabled to directly access a storage resource. The service implements a mechanism that generates an “impersonation token” by authorizing long running/background production jobs to access data without requiring further user interactions. \ No newline at end of file -- GitLab From db18ab294708e0a46bac050fd9e70c0cfb23656d Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Tue, 31 Aug 2021 16:24:59 -0500 Subject: [PATCH 25/45] ci: updating trigger in build --- app/sdms/devops/osdu/build/seismic-store-service.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/sdms/devops/osdu/build/seismic-store-service.yml b/app/sdms/devops/osdu/build/seismic-store-service.yml index 0aaf98f7..14f54cc7 100644 --- a/app/sdms/devops/osdu/build/seismic-store-service.yml +++ b/app/sdms/devops/osdu/build/seismic-store-service.yml @@ -14,5 +14,7 @@ compile-and-unit-test: - chmod +x devops/scripts/utest_build_run.sh - devops/scripts/utest_build_run.sh only: + refs: + - merge_requests changes: - app/sdms/**/* -- GitLab From c71f94da144e2fbb21f4acb8706a52f26cd9b49c Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Wed, 1 Sep 2021 08:23:09 -0500 Subject: [PATCH 26/45] ci: updating rules in ci cd pipeline --- app/sdms/.gitlab-ci.yml | 23 +------------------ .../osdu/build/seismic-store-service.yml | 4 +--- .../aws-seismic-store-service.yml | 2 ++ .../devops/osdu/scanners/fossa-global.yml | 9 ++++++-- 4 files changed, 11 insertions(+), 27 deletions(-) diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml index 9fc43c53..87e55cb4 100644 --- a/app/sdms/.gitlab-ci.yml +++ b/app/sdms/.gitlab-ci.yml @@ -17,25 +17,4 @@ include: # fossa - local: "app/sdms/devops/osdu/scanners/fossa-node.yml" - # lint - - local: "app/sdms/devops/osdu/scanners/lint-node.yml" - - # scan for secrets - - local: "app/sdms/devops/osdu/scanners/scan-for-secrets-node.yml" - - # containerize - # osdu - - project: "app/sdms/devops/osdu/contenerize/seismic-store-service.yml" - - # deploy - # azure - - local: "app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml" - - # aws - - local: "app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml" - - # gcp - - local: "app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml" - - # ibm - - local: "app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml" \ No newline at end of file + \ No newline at end of file diff --git a/app/sdms/devops/osdu/build/seismic-store-service.yml b/app/sdms/devops/osdu/build/seismic-store-service.yml index 14f54cc7..48f2a472 100644 --- a/app/sdms/devops/osdu/build/seismic-store-service.yml +++ b/app/sdms/devops/osdu/build/seismic-store-service.yml @@ -13,8 +13,6 @@ compile-and-unit-test: - cd app/sdms - chmod +x devops/scripts/utest_build_run.sh - devops/scripts/utest_build_run.sh - only: - refs: - - merge_requests + rules: changes: - app/sdms/**/* diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 066daf08..22cd71d7 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -195,6 +195,8 @@ aws-containerize: only: variables: - $AWS == 'true' + refs: + - merge_requests changes: - app/sdms/**/* diff --git a/app/sdms/devops/osdu/scanners/fossa-global.yml b/app/sdms/devops/osdu/scanners/fossa-global.yml index fc92179c..454d2383 100644 --- a/app/sdms/devops/osdu/scanners/fossa-global.yml +++ b/app/sdms/devops/osdu/scanners/fossa-global.yml @@ -1,10 +1,14 @@ - # This checks to see if the NOTICE file generated based on the current code # matches the one that is committed in the repository. Because FOSSA can # occassionally order the lines different between executions, we use a python # program to parse the file into sections and sort each section. Then we compare # and expect the same number of sections, names of sections, and contents # (without consider order). + +include: + - project: "osdu/platform/ci-cd-pipelines" + file: "scanners/fossa-global.yml" + fossa-check-notice: image: $CI_REGISTRY/divido/fossa-cli/fossa-cli:v6.1 stage: attribution @@ -13,9 +17,9 @@ fossa-check-notice: only: variables: - $FOSSA_API_KEY + rules: changes: - app/sdms/**/* - artifacts: when: on_failure paths: @@ -122,6 +126,7 @@ fossa-report: # To get the FOSSA_ACCESS, log into the project's main page and generate a "Live Project Report" URL from the Summary tab # The access token will be shown in the URL (the last path component) - $FOSSA_API_KEY && $FOSSA_ACCESS + rules: changes: - app/sdms/**/* -- GitLab From 134a6eb36c23da637fbd60d2f83f2a5af86743dd Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Wed, 1 Sep 2021 08:24:26 -0500 Subject: [PATCH 27/45] ci: updating rules in ci cd pipeline --- app/sdms/devops/osdu/build/seismic-store-service.yml | 2 +- app/sdms/devops/osdu/scanners/fossa-global.yml | 4 ++-- app/sdms/devops/osdu/scanners/fossa-node.yml | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/app/sdms/devops/osdu/build/seismic-store-service.yml b/app/sdms/devops/osdu/build/seismic-store-service.yml index 48f2a472..91ad78c1 100644 --- a/app/sdms/devops/osdu/build/seismic-store-service.yml +++ b/app/sdms/devops/osdu/build/seismic-store-service.yml @@ -14,5 +14,5 @@ compile-and-unit-test: - chmod +x devops/scripts/utest_build_run.sh - devops/scripts/utest_build_run.sh rules: - changes: + - changes: - app/sdms/**/* diff --git a/app/sdms/devops/osdu/scanners/fossa-global.yml b/app/sdms/devops/osdu/scanners/fossa-global.yml index 454d2383..4ec7bb93 100644 --- a/app/sdms/devops/osdu/scanners/fossa-global.yml +++ b/app/sdms/devops/osdu/scanners/fossa-global.yml @@ -18,7 +18,7 @@ fossa-check-notice: variables: - $FOSSA_API_KEY rules: - changes: + - changes: - app/sdms/**/* artifacts: when: on_failure @@ -127,7 +127,7 @@ fossa-report: # The access token will be shown in the URL (the last path component) - $FOSSA_API_KEY && $FOSSA_ACCESS rules: - changes: + - changes: - app/sdms/**/* variables: diff --git a/app/sdms/devops/osdu/scanners/fossa-node.yml b/app/sdms/devops/osdu/scanners/fossa-node.yml index 1e4a74d5..0c872b46 100644 --- a/app/sdms/devops/osdu/scanners/fossa-node.yml +++ b/app/sdms/devops/osdu/scanners/fossa-node.yml @@ -20,5 +20,6 @@ fossa-analyze: only: variables: - $FOSSA_API_KEY - changes: + rules: + - changes: - app/sdms/**/* -- GitLab From 26c5b047fc389e7af480026fe98be51cbe876e57 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Wed, 1 Sep 2021 08:33:19 -0500 Subject: [PATCH 28/45] ci: updating rules in ci cd pipeline --- app/sdms/devops/osdu/build/seismic-store-service.yml | 7 +++++-- app/sdms/devops/osdu/scanners/fossa-global.yml | 12 ++++++++---- app/sdms/devops/osdu/scanners/fossa-node.yml | 6 ++++-- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/app/sdms/devops/osdu/build/seismic-store-service.yml b/app/sdms/devops/osdu/build/seismic-store-service.yml index 91ad78c1..9a2a9c5c 100644 --- a/app/sdms/devops/osdu/build/seismic-store-service.yml +++ b/app/sdms/devops/osdu/build/seismic-store-service.yml @@ -13,6 +13,9 @@ compile-and-unit-test: - cd app/sdms - chmod +x devops/scripts/utest_build_run.sh - devops/scripts/utest_build_run.sh - rules: - - changes: + only: + refs: + - main + - merge_requests + changes: - app/sdms/**/* diff --git a/app/sdms/devops/osdu/scanners/fossa-global.yml b/app/sdms/devops/osdu/scanners/fossa-global.yml index 4ec7bb93..1434db92 100644 --- a/app/sdms/devops/osdu/scanners/fossa-global.yml +++ b/app/sdms/devops/osdu/scanners/fossa-global.yml @@ -17,8 +17,10 @@ fossa-check-notice: only: variables: - $FOSSA_API_KEY - rules: - - changes: + refs: + - main + - merge_requests + changes: - app/sdms/**/* artifacts: when: on_failure @@ -126,8 +128,10 @@ fossa-report: # To get the FOSSA_ACCESS, log into the project's main page and generate a "Live Project Report" URL from the Summary tab # The access token will be shown in the URL (the last path component) - $FOSSA_API_KEY && $FOSSA_ACCESS - rules: - - changes: + refs: + - main + - merge_requests + changes: - app/sdms/**/* variables: diff --git a/app/sdms/devops/osdu/scanners/fossa-node.yml b/app/sdms/devops/osdu/scanners/fossa-node.yml index 0c872b46..8fdd75cb 100644 --- a/app/sdms/devops/osdu/scanners/fossa-node.yml +++ b/app/sdms/devops/osdu/scanners/fossa-node.yml @@ -20,6 +20,8 @@ fossa-analyze: only: variables: - $FOSSA_API_KEY - rules: - - changes: + refs: + - main + - merge_requests + changes: - app/sdms/**/* -- GitLab From 258f5b5f9f15948bcc380fcaaff3d84507000cd3 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Wed, 1 Sep 2021 08:34:50 -0500 Subject: [PATCH 29/45] ci: updating rules in ci cd pipeline --- app/sdms/devops/osdu/build/seismic-store-service.yml | 4 ++-- app/sdms/devops/osdu/scanners/fossa-global.yml | 8 ++++---- app/sdms/devops/osdu/scanners/fossa-node.yml | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/app/sdms/devops/osdu/build/seismic-store-service.yml b/app/sdms/devops/osdu/build/seismic-store-service.yml index 9a2a9c5c..7ac85dfb 100644 --- a/app/sdms/devops/osdu/build/seismic-store-service.yml +++ b/app/sdms/devops/osdu/build/seismic-store-service.yml @@ -14,8 +14,8 @@ compile-and-unit-test: - chmod +x devops/scripts/utest_build_run.sh - devops/scripts/utest_build_run.sh only: + changes: + - app/sdms/**/* refs: - main - merge_requests - changes: - - app/sdms/**/* diff --git a/app/sdms/devops/osdu/scanners/fossa-global.yml b/app/sdms/devops/osdu/scanners/fossa-global.yml index 1434db92..ad9b3d2b 100644 --- a/app/sdms/devops/osdu/scanners/fossa-global.yml +++ b/app/sdms/devops/osdu/scanners/fossa-global.yml @@ -17,11 +17,11 @@ fossa-check-notice: only: variables: - $FOSSA_API_KEY + changes: + - app/sdms/**/* refs: - main - merge_requests - changes: - - app/sdms/**/* artifacts: when: on_failure paths: @@ -128,11 +128,11 @@ fossa-report: # To get the FOSSA_ACCESS, log into the project's main page and generate a "Live Project Report" URL from the Summary tab # The access token will be shown in the URL (the last path component) - $FOSSA_API_KEY && $FOSSA_ACCESS + changes: + - app/sdms/**/* refs: - main - merge_requests - changes: - - app/sdms/**/* variables: FOSSA_REPORT_URL: https://app.fossa.com/api/revisions/custom%2B12773%2F${CI_PROJECT_NAME}%24${CI_COMMIT_SHA}/attribution?access=${FOSSA_ACCESS}&includeProjectLicense=true&includeLicenseScan=true&includeDirectDependencies=true&includeDeepDependencies=true&includeLicenseHeaders=&includeLicenseList=true&format=HTML diff --git a/app/sdms/devops/osdu/scanners/fossa-node.yml b/app/sdms/devops/osdu/scanners/fossa-node.yml index 8fdd75cb..a44d3dcb 100644 --- a/app/sdms/devops/osdu/scanners/fossa-node.yml +++ b/app/sdms/devops/osdu/scanners/fossa-node.yml @@ -20,8 +20,8 @@ fossa-analyze: only: variables: - $FOSSA_API_KEY + changes: + - app/sdms/**/* refs: - main - merge_requests - changes: - - app/sdms/**/* -- GitLab From 7e9e2e87ed39c87ed98c744e81f32044ba92fa25 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Wed, 1 Sep 2021 08:36:17 -0500 Subject: [PATCH 30/45] ci: updating rules in ci cd pipeline --- app/sdms/devops/osdu/build/seismic-store-service.yml | 1 + app/sdms/devops/osdu/scanners/fossa-global.yml | 2 ++ app/sdms/devops/osdu/scanners/fossa-node.yml | 1 + 3 files changed, 4 insertions(+) diff --git a/app/sdms/devops/osdu/build/seismic-store-service.yml b/app/sdms/devops/osdu/build/seismic-store-service.yml index 7ac85dfb..91a7d639 100644 --- a/app/sdms/devops/osdu/build/seismic-store-service.yml +++ b/app/sdms/devops/osdu/build/seismic-store-service.yml @@ -17,5 +17,6 @@ compile-and-unit-test: changes: - app/sdms/**/* refs: + - branches - main - merge_requests diff --git a/app/sdms/devops/osdu/scanners/fossa-global.yml b/app/sdms/devops/osdu/scanners/fossa-global.yml index ad9b3d2b..9db2e3ee 100644 --- a/app/sdms/devops/osdu/scanners/fossa-global.yml +++ b/app/sdms/devops/osdu/scanners/fossa-global.yml @@ -20,6 +20,7 @@ fossa-check-notice: changes: - app/sdms/**/* refs: + - branches - main - merge_requests artifacts: @@ -131,6 +132,7 @@ fossa-report: changes: - app/sdms/**/* refs: + - branches - main - merge_requests diff --git a/app/sdms/devops/osdu/scanners/fossa-node.yml b/app/sdms/devops/osdu/scanners/fossa-node.yml index a44d3dcb..c0e57ea3 100644 --- a/app/sdms/devops/osdu/scanners/fossa-node.yml +++ b/app/sdms/devops/osdu/scanners/fossa-node.yml @@ -23,5 +23,6 @@ fossa-analyze: changes: - app/sdms/**/* refs: + - branches - main - merge_requests -- GitLab From 9607cd11839b4a1fb8c1e18ff1e709ffb56c88a5 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Wed, 1 Sep 2021 09:36:10 -0500 Subject: [PATCH 31/45] ci: updating rules in ci cd pipeline --- app/sdms/.gitlab-ci.yml | 25 +- .../osdu/build/seismic-store-service.yml | 4 +- .../aws-seismic-store-service.yml | 176 ++------------ .../azure-seismic-store-service.yml | 16 +- .../gcp-seismic-store-service.yml | 230 ++++++------------ .../ibm-seismic-store-service.yml | 40 +-- .../contenerize/seismic-store-service.yml | 6 +- app/sdms/devops/osdu/scanners/lint-node.yml | 6 +- .../osdu/scanners/scan-for-secrets-node.yml | 6 +- 9 files changed, 155 insertions(+), 354 deletions(-) diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml index 87e55cb4..59342f9f 100644 --- a/app/sdms/.gitlab-ci.yml +++ b/app/sdms/.gitlab-ci.yml @@ -5,8 +5,6 @@ variables: PORT: 80 REPLICA: 1 UTEST_RUNTIME_IMAGE: seistore-svc-runtime - SDMS_MIN_REPLICAS: 1 - SDMS_MAX_REPLICAS: 5 include: @@ -17,4 +15,25 @@ include: # fossa - local: "app/sdms/devops/osdu/scanners/fossa-node.yml" - \ No newline at end of file + # lint + - local: "app/sdms/devops/osdu/scanners/lint-node.yml" + + # scan for secrets + - local: "app/sdms/devops/osdu/scanners/scan-for-secrets-node.yml" + + # containerize + # osdu + - project: "app/sdms/devops/osdu/contenerize/seismic-store-service.yml" + + # deploy + # azure + - local: "app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml" + + # aws + - local: "app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml" + + # gcp + - local: "app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml" + + # ibm + - local: "app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml" \ No newline at end of file diff --git a/app/sdms/devops/osdu/build/seismic-store-service.yml b/app/sdms/devops/osdu/build/seismic-store-service.yml index 91a7d639..40fec8ea 100644 --- a/app/sdms/devops/osdu/build/seismic-store-service.yml +++ b/app/sdms/devops/osdu/build/seismic-store-service.yml @@ -5,9 +5,9 @@ compile-and-unit-test: artifacts: reports: junit: test-results.xml - cobertura: coverage/cobertura-coverage.xml + cobertura: app/sdms/coverage/cobertura-coverage.xml paths: - - coverage/ + - app/sdms/coverage/ expire_in: 15 days script: - cd app/sdms diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 22cd71d7..7d3ffe20 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -1,3 +1,7 @@ +include: + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/aws-global.yml" + variables: #aws variables AWS_SERVICE: seismic-store @@ -11,160 +15,6 @@ variables: # skipping tests here. Using a local file to run tests AWS_SKIP_TESTS: "true" -.aws_variables: - - before_script: - - | - if [ $AWS_DEPLOY_TARGET == 'EKS' ]; then - - echo "##### DEPLOYING TO EKS/GREEN ######" - - export AWS_INFRA_STACK_NAME="$AWS_GREEN_INFRA_STACK_NAME"; - export AWS_REGION="$AWS_GREEN_REGION"; - export AWS_DOMAIN="${AWS_GREEN_SUBDOMAIN}.${AWS_BASE_DOMAIN}"; - export RESOURCE_PREFIX="$AWS_GREEN_RESOURCE_PREFIX"; - - else - - echo "##### DEPLOYING TO ECS/BLUE ######" - - echo "FAILED>>>BLUE ENVIRONMENT DOES NOT EXIST" - - exit 1 - - export AWS_INFRA_STACK_NAME="$AWS_BLUE_INFRA_STACK_NAME"; - export AWS_REGION="$AWS_BLUE_REGION"; - export AWS_DOMAIN="${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}"; - export RESOURCE_PREFIX="$AWS_BLUE_RESOURCE_PREFIX"; - - fi - - export S3_LEGAL_CONFIG_BUCKET=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/shared/legal/s3DataBucket/name --query Parameter.Value --output text --region $AWS_REGION); - export AWS_COGNITO_CLIENT_ID=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/cognito-client-id --query Parameter.Value --output text --region $AWS_REGION); - export AWS_COGNITO_USER_POOL_ID=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/cognito-user-pool-id --query Parameter.Value --output text --region $AWS_REGION); - - export ELASTIC_HOST=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/elasticsearch/end-point --query Parameter.Value --output text --region $AWS_REGION); - export ELASTIC_SECRET_STRING=$(aws secretsmanager get-secret-value --secret-id /osdu/${RESOURCE_PREFIX}/elasticsearch/credentials --query SecretString --output json --region $AWS_REGION | sed -e 's/\\\"/\"/g' -e 's/^.//g' -e 's/.$//g'); - export ELASTIC_USER_NAME=$(echo $ELASTIC_SECRET_STRING | jq -r '.username'); - export ELASTIC_PASSWORD=$(echo $ELASTIC_SECRET_STRING | jq -r '.password'); - - export AWS_BASE_URL="https://${AWS_DOMAIN}"; - export HOST_URL="${AWS_BASE_URL}/api/legal/v1/" - export LEGALTAG_BASE_URL="${AWS_DOMAIN}" - export STORAGE_URL="${AWS_BASE_URL}/api/storage/v2/" - export LEGAL_URL="${AWS_BASE_URL}/api/legal/v1/" - export SEARCH_HOST="${AWS_BASE_URL}/api/search/v2/" - export STORAGE_HOST="${AWS_BASE_URL}/api/storage/v2/" - export LEGAL_HOST="${AWS_BASE_URL}/api/legal/v1/" - export DELIVERY_HOST="${AWS_BASE_URL}/api/delivery/v2/" - export INDEXER_HOST="${AWS_BASE_URL}/api/indexer/v2/" - export ENTITLEMENTS_URL="${AWS_BASE_URL}/api/entitlements/v2/" - export VIRTUAL_SERVICE_HOST_NAME="${AWS_DOMAIN}" - export HOST="${AWS_BASE_URL}" - # needs trailing slash - export PARTITION_BASE_URL="${AWS_BASE_URL}/" - export WORKFLOW_HOST="${AWS_BASE_URL}/api/workflow/" - export DATA_WORKFLOW_HOST="${AWS_BASE_URL}/api/data-workflow/v1" - export REGISTER_CUSTOM_PUSH_URL="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge" - export REGISTER_CUSTOM_PUSH_URL1="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge/1" - # needs trailing slash - export REGISTER_BASE_URL="${AWS_BASE_URL}/" - export REGISTER_CUSTOM_PUSH_URL_HMAC="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge/1" - export NOTIFICATION_BASE_URL="${AWS_BASE_URL}/api/notification/v1/" - export NOTIFICATION_REGISTER_BASE_URL="${AWS_BASE_URL}" - export FILE_SERVICE_HOST="${AWS_BASE_URL}/api/file/v2" - export STORAGE_BASE_URL="${AWS_BASE_URL}/api/storage/v2/" - export DATASET_BASE_URL="${AWS_BASE_URL}/api/dataset/v1/" - export LEGAL_BASE_URL="${AWS_BASE_URL}/api/legal/v1/" - export ENTITLEMENTS_BASE_URL="${AWS_BASE_URL}/api/entitlements/v2/" - export FILEDMS_BASE_URL="${AWS_BASE_URL}/api/dms/file/v1/" - export SEISMICSTORE_SVC_URL="${AWS_BASE_URL}/api/seismic-store/v3" - export WELLBORE_DDMS_URL="${AWS_BASE_URL}/api/os-wellbore-ddms" - export EDSDMS_BASE_URL="${AWS_BASE_URL}/api/dms/eds/v1/" - export SCHEMA_BASE_URL="${AWS_BASE_URL}/api/schema-service/v1/" - export ENTITLEMENT_V2_URL="${AWS_BASE_URL}/api/entitlements/v2/" - export LEGAL_QUEUE="https://sqs.${AWS_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${RESOURCE_PREFIX}-legal-queue" - - export AWS_COGNITO_REGION="$AWS_REGION" - export AWS_S3_ENDPOINT="s3.${AWS_REGION}.amazonaws.com" - export AWS_S3_REGION="$AWS_REGION" - - export DYNAMO_DB_REGION="$AWS_REGION" - export DYNAMO_DB_ENDPOINT="dynamodb.${AWS_REGION}.amazonaws.com" - - export TABLE_PREFIX="$RESOURCE_PREFIX" - export AWS_CLUSTER_NAME="${RESOURCE_PREFIX}-core-cluster" - - echo "#### USING OSDU BASE URL: $AWS_BASE_URL ####" - -.aws_common_variables: - variables: - ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID - SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY - AWS_ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY - INTEGRATION_TEST_DIR: $AWS_TEST_SUBDIR - SERVICE_NAME: $AWS_SERVICE - BUILD_DIR: $AWS_BUILD_SUBDIR - ENVIRONMENT: $AWS_ENVIRONMENT - APPLICATION_NAME: os-$AWS_SERVICE - LOCAL_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA - AWS_IMAGE_TAG_BASE: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_ECR_REGION}.amazonaws.com/os-${AWS_SERVICE} - OTHER_RELEVANT_DATA_COUNTRIES: $AWS_OTHER_RELEVANT_DATA_COUNTRIES - LEGAL_TAG : $AWS_LEGAL_TAG - TENANT_NAME : $AWS_TENANT_NAME - PRIVATE_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1 - PRIVATE_TENANT2: tenant2 - SHARED_TENANT: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2 - VENDOR: aws - DOMAIN: example.com - AWS_COGNITO_AUTH_FLOW: $AWS_COGNITO_AUTH_FLOW - AWS_COGNITO_AUTH_PARAMS_PASSWORD: $AWS_COGNITO_AUTH_PARAMS_PASSWORD - AWS_COGNITO_AUTH_PARAMS_USER: $AWS_COGNITO_AUTH_PARAMS_USER - AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS: $AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS - AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID - DEPLOY_ENV: empty - MY_TENANT: $AWS_TENANT_NAME - ENTITLEMENTS_TEST_TENANT: $AWS_ENTITLEMENTS_TEST_TENANT - LOG_LEVEL: INFO - SKIP_HTTP_TESTS: 'true' - DELIVERY_INT_TEST_BUCKET_NAME: $AWS_DELIVERY_INT_TEST_BUCKET_NAME - DEFAULT_DATA_PARTITION_ID_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1 - DEFAULT_DATA_PARTITION_ID_TENANT2: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2 - #used by file->delivery int tests - DATA_PARTITION_ID: int-test-file - ENTITLEMENTS_DOMAIN: example.com - ELASTIC_PORT: 9200 - TEST_DAG_NAME: my_first_dag - SUBSCRIBER_SECRET: $AWS_SUBSCRIBER_SECRET - HMAC_SECRET: $AWS_HMAC_SECRET - PROVIDER_KEY: AWS_S3 - #File Service Variables - TIME_ZONE: UTC - USER_ID: $AWS_COGNITO_AUTH_PARAMS_USER - #Entitlements V2 variables--start-- - TENANT: opendes - SERVICE_PRINCIPAL_EMAIL: serviceprincipal@testing.com - #Entitlements V2 variables--end-- - - -.aws: - tags: ['osdu-medium'] - image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-maven/aws-maven:v1.3 - environment: - name: AWS - extends: - - .aws_common_variables - - .aws_variables - before_script: - - mkdir -p ~/.aws - - | - cat > ~/.aws/credentials < Date: Wed, 1 Sep 2021 09:50:46 -0500 Subject: [PATCH 32/45] ci: updating readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 58f27cac..71d4aecb 100644 --- a/README.md +++ b/README.md @@ -6,4 +6,4 @@ Saving a dataset on a cloud-based storage, as single entity, may be a problem wh Seismic Store is a cloud-based solution composed by restful micro-services, client APIs and tools designed to implement a multi-object storage approach. The system saves objects that compose a dataset as a hierarchical data structure in a cloud storage and the dataset properties as a metadata entry in a no-relational catalogue. Having the datasets stored as multiple independent objects improve the overall performance, as generic I/O operations, for example read or write objects, can be easily parallelized. -Seismic Store manages data authorization at service level by protecting access to storage bucket resources. Only service authorized users are enabled to directly access a storage resource. The service implements a mechanism that generates an “impersonation token” by authorizing long running/background production jobs to access data without requiring further user interactions. \ No newline at end of file +Seismic Store manages data authorization at service level by protecting access to storage bucket resources. Only service authorized users are enabled to directly access a storage resource. The service implements a mechanism that generates an “impersonation token” by authorizing long running/background production jobs to access data without requiring further user interactions. -- GitLab From 4418cc40c053e0ef1ad9f494123668290d6fcb73 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Thu, 2 Sep 2021 07:59:54 -0500 Subject: [PATCH 33/45] ci: updating fossa scan --- app/sdms/devops/osdu/scanners/fossa-global.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/sdms/devops/osdu/scanners/fossa-global.yml b/app/sdms/devops/osdu/scanners/fossa-global.yml index 9db2e3ee..55472fab 100644 --- a/app/sdms/devops/osdu/scanners/fossa-global.yml +++ b/app/sdms/devops/osdu/scanners/fossa-global.yml @@ -26,7 +26,7 @@ fossa-check-notice: artifacts: when: on_failure paths: - - public + - app/sdms/public script: # fossa-check-for-licensing-issues needs a CI_COMMIT_BRANCH defined to know how to parse the FOSSA API results -- GitLab From 170843ae37b3866b36caefff9a17d482f5923201 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Thu, 2 Sep 2021 08:11:08 -0500 Subject: [PATCH 34/45] ci: updating notice file --- app/sdms/NOTICE | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/sdms/NOTICE b/app/sdms/NOTICE index e3794b0e..26d5245f 100644 --- a/app/sdms/NOTICE +++ b/app/sdms/NOTICE @@ -48,7 +48,7 @@ The following software have components provided under the terms of this license: - aws-sign2 (from https://www.npmjs.com/package/aws-sign2) - boto3 (from https://github.com/boto/boto3) - botocore (from https://github.com/boto/botocore) -- browser-request (from http://github.com/iriscouch/browser-request) +- browser-request (from https://www.npmjs.com/package/browser-request) - caseless (from https://www.npmjs.com/package/caseless) - cloudant-follow (from https://www.npmjs.com/package/cloudant-follow) - cluster-key-slot (from https://www.npmjs.com/package/cluster-key-slot) @@ -135,7 +135,7 @@ The following software have components provided under the terms of this license: - qs (from https://www.npmjs.com/package/qs) - qs (from https://www.npmjs.com/package/qs) - rc (from https://www.npmjs.com/package/rc) -- shimmer (from https://github.com/othiym23/shimmer#readme) +- shimmer (from https://www.npmjs.com/package/shimmer) - source-map (from https://www.npmjs.com/package/source-map) - sprintf-js (from https://github.com/alexei/sprintf.js#readme) - tough-cookie (from https://www.npmjs.com/package/tough-cookie) @@ -193,7 +193,7 @@ The following software have components provided under the terms of this license: - tough-cookie (from https://www.npmjs.com/package/tough-cookie) - tslib (from https://www.npmjs.com/package/tslib) - uri-js (from https://www.npmjs.com/package/uri-js) -- url-template (from https://github.com/bramstein/url-template) +- url-template (from https://www.npmjs.com/package/url-template) - uuid (from https://www.npmjs.com/package/uuid) - uuid (from https://www.npmjs.com/package/uuid) - uuid (from https://www.npmjs.com/package/uuid) @@ -431,7 +431,7 @@ The following software have components provided under the terms of this license: - body-parser (from https://www.npmjs.com/package/body-parser) - botocore (from https://github.com/boto/botocore) - brace-expansion (from https://www.npmjs.com/package/brace-expansion) -- browser-request (from http://github.com/iriscouch/browser-request) +- browser-request (from https://www.npmjs.com/package/browser-request) - buffer (from https://www.npmjs.com/package/buffer) - buffer (from https://www.npmjs.com/package/buffer) - buffer-from (from https://www.npmjs.com/package/buffer-from) -- GitLab From 175414024d8a008e68d9ad05c3aca707972076c3 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Thu, 2 Sep 2021 08:42:49 -0500 Subject: [PATCH 35/45] ci: updating aws steps --- .../devops/osdu/cloud-providers/aws-seismic-store-service.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 7d3ffe20..d6b91fe2 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -61,6 +61,7 @@ aws-update-eks: stage: deploy needs: ['aws-containerize'] script: + - cd app/sdms - export KUBECONFIG=/tmp/kubeconfig-${RANDOM}.yaml - export EKS_CLUSTER_MGMT_ROLE=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterManagementRole'].OutputValue" --output text) - export EKS_CLUSTER_NAME=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterName'].OutputValue" --output text) -- GitLab From 895c6dbb04c9ee96867e76e27128a9216a852b81 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Thu, 2 Sep 2021 14:26:19 -0500 Subject: [PATCH 36/45] ci: fixing aws script issue --- .../aws-seismic-store-service.yml | 85 +++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index d6b91fe2..187b731d 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -2,6 +2,91 @@ include: - project: "osdu/platform/ci-cd-pipelines" file: "cloud-providers/aws-global.yml" +before_script: + - cd app/sdms + - | + if [ $AWS_DEPLOY_TARGET == 'EKS' ]; then + + echo "##### DEPLOYING TO EKS/GREEN ######" + + export AWS_INFRA_STACK_NAME="$AWS_GREEN_INFRA_STACK_NAME"; + export AWS_REGION="$AWS_GREEN_REGION"; + export AWS_DOMAIN="${AWS_GREEN_SUBDOMAIN}.${AWS_BASE_DOMAIN}"; + export RESOURCE_PREFIX="$AWS_GREEN_RESOURCE_PREFIX"; + + else + + echo "##### DEPLOYING TO ECS/BLUE ######" + + echo "FAILED>>>BLUE ENVIRONMENT DOES NOT EXIST" + + exit 1 + + export AWS_INFRA_STACK_NAME="$AWS_BLUE_INFRA_STACK_NAME"; + export AWS_REGION="$AWS_BLUE_REGION"; + export AWS_DOMAIN="${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}"; + export RESOURCE_PREFIX="$AWS_BLUE_RESOURCE_PREFIX"; + + fi + + export S3_LEGAL_CONFIG_BUCKET=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/shared/legal/s3DataBucket/name --query Parameter.Value --output text --region $AWS_REGION); + export AWS_COGNITO_CLIENT_ID=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/cognito-client-id --query Parameter.Value --output text --region $AWS_REGION); + export AWS_COGNITO_USER_POOL_ID=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/cognito-user-pool-id --query Parameter.Value --output text --region $AWS_REGION); + + export ELASTIC_HOST=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/elasticsearch/end-point --query Parameter.Value --output text --region $AWS_REGION); + export ELASTIC_SECRET_STRING=$(aws secretsmanager get-secret-value --secret-id /osdu/${RESOURCE_PREFIX}/elasticsearch/credentials --query SecretString --output json --region $AWS_REGION | sed -e 's/\\\"/\"/g' -e 's/^.//g' -e 's/.$//g'); + export ELASTIC_USER_NAME=$(echo $ELASTIC_SECRET_STRING | jq -r '.username'); + export ELASTIC_PASSWORD=$(echo $ELASTIC_SECRET_STRING | jq -r '.password'); + + export AWS_BASE_URL="https://${AWS_DOMAIN}"; + export HOST_URL="${AWS_BASE_URL}/api/legal/v1/" + export LEGALTAG_BASE_URL="${AWS_DOMAIN}" + export STORAGE_URL="${AWS_BASE_URL}/api/storage/v2/" + export LEGAL_URL="${AWS_BASE_URL}/api/legal/v1/" + export SEARCH_HOST="${AWS_BASE_URL}/api/search/v2/" + export STORAGE_HOST="${AWS_BASE_URL}/api/storage/v2/" + export LEGAL_HOST="${AWS_BASE_URL}/api/legal/v1/" + export DELIVERY_HOST="${AWS_BASE_URL}/api/delivery/v2/" + export INDEXER_HOST="${AWS_BASE_URL}/api/indexer/v2/" + export ENTITLEMENTS_URL="${AWS_BASE_URL}/api/entitlements/v2/" + export VIRTUAL_SERVICE_HOST_NAME="${AWS_DOMAIN}" + export HOST="${AWS_BASE_URL}" + # needs trailing slash + export PARTITION_BASE_URL="${AWS_BASE_URL}/" + export WORKFLOW_HOST="${AWS_BASE_URL}/api/workflow/" + export DATA_WORKFLOW_HOST="${AWS_BASE_URL}/api/data-workflow/v1" + export REGISTER_CUSTOM_PUSH_URL="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge" + export REGISTER_CUSTOM_PUSH_URL1="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge/1" + # needs trailing slash + export REGISTER_BASE_URL="${AWS_BASE_URL}/" + export REGISTER_CUSTOM_PUSH_URL_HMAC="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge/1" + export NOTIFICATION_BASE_URL="${AWS_BASE_URL}/api/notification/v1/" + export NOTIFICATION_REGISTER_BASE_URL="${AWS_BASE_URL}" + export FILE_SERVICE_HOST="${AWS_BASE_URL}/api/file/v2" + export STORAGE_BASE_URL="${AWS_BASE_URL}/api/storage/v2/" + export DATASET_BASE_URL="${AWS_BASE_URL}/api/dataset/v1/" + export LEGAL_BASE_URL="${AWS_BASE_URL}/api/legal/v1/" + export ENTITLEMENTS_BASE_URL="${AWS_BASE_URL}/api/entitlements/v2/" + export FILEDMS_BASE_URL="${AWS_BASE_URL}/api/dms/file/v1/" + export SEISMICSTORE_SVC_URL="${AWS_BASE_URL}/api/seismic-store/v3" + export WELLBORE_DDMS_URL="${AWS_BASE_URL}/api/os-wellbore-ddms" + export EDSDMS_BASE_URL="${AWS_BASE_URL}/api/dms/eds/v1/" + export SCHEMA_BASE_URL="${AWS_BASE_URL}/api/schema-service/v1/" + export ENTITLEMENT_V2_URL="${AWS_BASE_URL}/api/entitlements/v2/" + export LEGAL_QUEUE="https://sqs.${AWS_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${RESOURCE_PREFIX}-legal-queue" + + export AWS_COGNITO_REGION="$AWS_REGION" + export AWS_S3_ENDPOINT="s3.${AWS_REGION}.amazonaws.com" + export AWS_S3_REGION="$AWS_REGION" + + export DYNAMO_DB_REGION="$AWS_REGION" + export DYNAMO_DB_ENDPOINT="dynamodb.${AWS_REGION}.amazonaws.com" + + export TABLE_PREFIX="$RESOURCE_PREFIX" + export AWS_CLUSTER_NAME="${RESOURCE_PREFIX}-core-cluster" + + echo "#### USING OSDU BASE URL: $AWS_BASE_URL ####" + variables: #aws variables AWS_SERVICE: seismic-store -- GitLab From e210e9d477deeec51f6c0d0d4483cddc8d25de85 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Thu, 2 Sep 2021 14:33:06 -0500 Subject: [PATCH 37/45] ci: fixing aws script issue --- .../devops/osdu/cloud-providers/aws-seismic-store-service.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 187b731d..32edadc2 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -2,7 +2,8 @@ include: - project: "osdu/platform/ci-cd-pipelines" file: "cloud-providers/aws-global.yml" -before_script: +.aws_variables: + before_script: - cd app/sdms - | if [ $AWS_DEPLOY_TARGET == 'EKS' ]; then -- GitLab From 03addbdfce01d51c451e440d153a26a1608c9b1c Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Thu, 2 Sep 2021 14:52:44 -0500 Subject: [PATCH 38/45] ci: fixing aws script issue --- .../devops/osdu/cloud-providers/aws-seismic-store-service.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 32edadc2..339dc8e9 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -109,7 +109,6 @@ aws-containerize: stage: containerize needs: ['compile-and-unit-test'] script: - - cd app/sdms - | if [ -z $AWS_BUILDER_DOCKERFILE_PATH ] && [ -z $AWS_RUNTIME_DOCKERFILE_PATH ]; then echo Building BUILD_DIR/Dockerfile container @@ -147,7 +146,6 @@ aws-update-eks: stage: deploy needs: ['aws-containerize'] script: - - cd app/sdms - export KUBECONFIG=/tmp/kubeconfig-${RANDOM}.yaml - export EKS_CLUSTER_MGMT_ROLE=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterManagementRole'].OutputValue" --output text) - export EKS_CLUSTER_NAME=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterName'].OutputValue" --output text) @@ -199,7 +197,6 @@ aws-test-newman: image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-node/aws-node:v1.0-node14 needs: [{ job: 'aws-update-ecs', optional: true }, { job: 'aws-update-eks', optional: true }] script: - - cd app/sdms - pip3 install -r devops/aws/requirements.txt - svctoken=$(python3 devops/scripts/aws_jwt_client.py) - npm install -g newman -- GitLab From b8a6a27a4ae2c8acb7b39b4fcb8359bbc4396928 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Fri, 3 Sep 2021 08:35:03 -0500 Subject: [PATCH 39/45] ci: fixing aws script issue --- .../aws-seismic-store-service.yml | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 339dc8e9..bcc7cb6d 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -1,10 +1,9 @@ include: - project: "osdu/platform/ci-cd-pipelines" file: "cloud-providers/aws-global.yml" - .aws_variables: + before_script: - - cd app/sdms - | if [ $AWS_DEPLOY_TARGET == 'EKS' ]; then @@ -88,6 +87,25 @@ include: echo "#### USING OSDU BASE URL: $AWS_BASE_URL ####" +.aws: + tags: ['osdu-medium'] + image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-maven/aws-maven:v1.3 + environment: + name: AWS + extends: + - .aws_common_variables + - .aws_variables + before_script: + - mkdir -p ~/.aws + - | + cat > ~/.aws/credentials < Date: Fri, 3 Sep 2021 09:11:53 -0500 Subject: [PATCH 40/45] ci: fixing aws script issue --- .../aws-seismic-store-service.yml | 104 +++++++++++------- 1 file changed, 65 insertions(+), 39 deletions(-) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index bcc7cb6d..87ae7c3e 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -1,6 +1,20 @@ include: - project: "osdu/platform/ci-cd-pipelines" file: "cloud-providers/aws-global.yml" + +variables: + #aws variables + AWS_SERVICE: seismic-store + AWS_BUILD_SUBDIR: src/cloud/providers/aws/build-aws + AWS_TEST_SUBDIR: tests + AWS_ENVIRONMENT: dev + AWS_DEPLOY_TARGET: EKS + AWS_EKS_DEPLOYMENT_NAME: os-seismic-store + AWS_BUILDER_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/builder.Dockerfile + AWS_RUNTIME_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/runtime.Dockerfile + # skipping tests here. Using a local file to run tests + AWS_SKIP_TESTS: "true" + .aws_variables: before_script: @@ -87,6 +101,56 @@ include: echo "#### USING OSDU BASE URL: $AWS_BASE_URL ####" +.aws_common_variables: + variables: + ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID + SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY + AWS_ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID + AWS_SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY + INTEGRATION_TEST_DIR: $AWS_TEST_SUBDIR + SERVICE_NAME: $AWS_SERVICE + BUILD_DIR: $AWS_BUILD_SUBDIR + ENVIRONMENT: $AWS_ENVIRONMENT + APPLICATION_NAME: os-$AWS_SERVICE + LOCAL_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA + AWS_IMAGE_TAG_BASE: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_ECR_REGION}.amazonaws.com/os-${AWS_SERVICE} + OTHER_RELEVANT_DATA_COUNTRIES: $AWS_OTHER_RELEVANT_DATA_COUNTRIES + LEGAL_TAG : $AWS_LEGAL_TAG + TENANT_NAME : $AWS_TENANT_NAME + PRIVATE_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1 + PRIVATE_TENANT2: tenant2 + SHARED_TENANT: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2 + VENDOR: aws + DOMAIN: example.com + AWS_COGNITO_AUTH_FLOW: $AWS_COGNITO_AUTH_FLOW + AWS_COGNITO_AUTH_PARAMS_PASSWORD: $AWS_COGNITO_AUTH_PARAMS_PASSWORD + AWS_COGNITO_AUTH_PARAMS_USER: $AWS_COGNITO_AUTH_PARAMS_USER + AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS: $AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS + AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID + DEPLOY_ENV: empty + MY_TENANT: $AWS_TENANT_NAME + ENTITLEMENTS_TEST_TENANT: $AWS_ENTITLEMENTS_TEST_TENANT + LOG_LEVEL: INFO + SKIP_HTTP_TESTS: 'true' + DELIVERY_INT_TEST_BUCKET_NAME: $AWS_DELIVERY_INT_TEST_BUCKET_NAME + DEFAULT_DATA_PARTITION_ID_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1 + DEFAULT_DATA_PARTITION_ID_TENANT2: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2 + #used by file->delivery int tests + DATA_PARTITION_ID: int-test-file + ENTITLEMENTS_DOMAIN: example.com + ELASTIC_PORT: 9200 + TEST_DAG_NAME: my_first_dag + SUBSCRIBER_SECRET: $AWS_SUBSCRIBER_SECRET + HMAC_SECRET: $AWS_HMAC_SECRET + PROVIDER_KEY: AWS_S3 + #File Service Variables + TIME_ZONE: UTC + USER_ID: $AWS_COGNITO_AUTH_PARAMS_USER + #Entitlements V2 variables--start-- + TENANT: opendes + SERVICE_PRINCIPAL_EMAIL: serviceprincipal@testing.com + #Entitlements V2 variables--end-- + .aws: tags: ['osdu-medium'] image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-maven/aws-maven:v1.3 @@ -104,21 +168,6 @@ include: aws_secret_access_key = $SECRET_ACCESS_KEY EOF - - -variables: - #aws variables - AWS_SERVICE: seismic-store - AWS_BUILD_SUBDIR: src/cloud/providers/aws/build-aws - AWS_TEST_SUBDIR: tests - AWS_ENVIRONMENT: dev - AWS_DEPLOY_TARGET: EKS - AWS_EKS_DEPLOYMENT_NAME: os-seismic-store - AWS_BUILDER_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/builder.Dockerfile - AWS_RUNTIME_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/runtime.Dockerfile - # skipping tests here. Using a local file to run tests - AWS_SKIP_TESTS: "true" - aws-containerize: extends: - .aws @@ -156,7 +205,6 @@ aws-containerize: - main - merge_requests - aws-update-eks: extends: - .aws @@ -183,29 +231,6 @@ aws-update-eks: - branches - main - merge_requests - -aws-update-ecs: - extends: - - .aws - - .aws_common_variables - - .aws_variables - stage: deploy - needs: ['aws-containerize'] - script: - - ECS_SERVICE_NAME=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/ecs/services/${SERVICE_NAME} --query Parameter.Value --output text --region $AWS_REGION) - #limit output to the first 50 lines...the rest is bloat - - export ECS_UPDATE_TMP_FILE=/tmp/ecs-update-log-${RANDOM}.txt - - aws ecs update-service --cluster $AWS_CLUSTER_NAME --service $ECS_SERVICE_NAME --region $AWS_REGION --force-new-deployment > $ECS_UPDATE_TMP_FILE - - cat $ECS_UPDATE_TMP_FILE | head -n 50 - only: - variables: - - $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true' && ($AWS_DEPLOY_TARGET == 'ECS' || $AWS_DEPLOY_TARGET == null) - changes: - - app/sdms/**/* - refs: - - branches - - main - - merge_requests aws-test-newman: extends: @@ -216,6 +241,7 @@ aws-test-newman: image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-node/aws-node:v1.0-node14 needs: [{ job: 'aws-update-ecs', optional: true }, { job: 'aws-update-eks', optional: true }] script: + - cd app/sdms - pip3 install -r devops/aws/requirements.txt - svctoken=$(python3 devops/scripts/aws_jwt_client.py) - npm install -g newman -- GitLab From 3e68d89b4134b76ba5d9debeb010d8a6b326b37f Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Fri, 3 Sep 2021 11:06:33 -0500 Subject: [PATCH 41/45] ci: fixing aws script issue --- .../aws-seismic-store-service.yml | 153 ------------------ 1 file changed, 153 deletions(-) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 87ae7c3e..42a37263 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -15,159 +15,6 @@ variables: # skipping tests here. Using a local file to run tests AWS_SKIP_TESTS: "true" -.aws_variables: - - before_script: - - | - if [ $AWS_DEPLOY_TARGET == 'EKS' ]; then - - echo "##### DEPLOYING TO EKS/GREEN ######" - - export AWS_INFRA_STACK_NAME="$AWS_GREEN_INFRA_STACK_NAME"; - export AWS_REGION="$AWS_GREEN_REGION"; - export AWS_DOMAIN="${AWS_GREEN_SUBDOMAIN}.${AWS_BASE_DOMAIN}"; - export RESOURCE_PREFIX="$AWS_GREEN_RESOURCE_PREFIX"; - - else - - echo "##### DEPLOYING TO ECS/BLUE ######" - - echo "FAILED>>>BLUE ENVIRONMENT DOES NOT EXIST" - - exit 1 - - export AWS_INFRA_STACK_NAME="$AWS_BLUE_INFRA_STACK_NAME"; - export AWS_REGION="$AWS_BLUE_REGION"; - export AWS_DOMAIN="${AWS_BLUE_SUBDOMAIN}.${AWS_BASE_DOMAIN}"; - export RESOURCE_PREFIX="$AWS_BLUE_RESOURCE_PREFIX"; - - fi - - export S3_LEGAL_CONFIG_BUCKET=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/shared/legal/s3DataBucket/name --query Parameter.Value --output text --region $AWS_REGION); - export AWS_COGNITO_CLIENT_ID=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/cognito-client-id --query Parameter.Value --output text --region $AWS_REGION); - export AWS_COGNITO_USER_POOL_ID=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/cognito-user-pool-id --query Parameter.Value --output text --region $AWS_REGION); - - export ELASTIC_HOST=$(aws ssm get-parameter --name /osdu/${RESOURCE_PREFIX}/elasticsearch/end-point --query Parameter.Value --output text --region $AWS_REGION); - export ELASTIC_SECRET_STRING=$(aws secretsmanager get-secret-value --secret-id /osdu/${RESOURCE_PREFIX}/elasticsearch/credentials --query SecretString --output json --region $AWS_REGION | sed -e 's/\\\"/\"/g' -e 's/^.//g' -e 's/.$//g'); - export ELASTIC_USER_NAME=$(echo $ELASTIC_SECRET_STRING | jq -r '.username'); - export ELASTIC_PASSWORD=$(echo $ELASTIC_SECRET_STRING | jq -r '.password'); - - export AWS_BASE_URL="https://${AWS_DOMAIN}"; - export HOST_URL="${AWS_BASE_URL}/api/legal/v1/" - export LEGALTAG_BASE_URL="${AWS_DOMAIN}" - export STORAGE_URL="${AWS_BASE_URL}/api/storage/v2/" - export LEGAL_URL="${AWS_BASE_URL}/api/legal/v1/" - export SEARCH_HOST="${AWS_BASE_URL}/api/search/v2/" - export STORAGE_HOST="${AWS_BASE_URL}/api/storage/v2/" - export LEGAL_HOST="${AWS_BASE_URL}/api/legal/v1/" - export DELIVERY_HOST="${AWS_BASE_URL}/api/delivery/v2/" - export INDEXER_HOST="${AWS_BASE_URL}/api/indexer/v2/" - export ENTITLEMENTS_URL="${AWS_BASE_URL}/api/entitlements/v2/" - export VIRTUAL_SERVICE_HOST_NAME="${AWS_DOMAIN}" - export HOST="${AWS_BASE_URL}" - # needs trailing slash - export PARTITION_BASE_URL="${AWS_BASE_URL}/" - export WORKFLOW_HOST="${AWS_BASE_URL}/api/workflow/" - export DATA_WORKFLOW_HOST="${AWS_BASE_URL}/api/data-workflow/v1" - export REGISTER_CUSTOM_PUSH_URL="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge" - export REGISTER_CUSTOM_PUSH_URL1="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge/1" - # needs trailing slash - export REGISTER_BASE_URL="${AWS_BASE_URL}/" - export REGISTER_CUSTOM_PUSH_URL_HMAC="${AWS_BASE_URL}/api/register/v1/awstest/aws/challenge/1" - export NOTIFICATION_BASE_URL="${AWS_BASE_URL}/api/notification/v1/" - export NOTIFICATION_REGISTER_BASE_URL="${AWS_BASE_URL}" - export FILE_SERVICE_HOST="${AWS_BASE_URL}/api/file/v2" - export STORAGE_BASE_URL="${AWS_BASE_URL}/api/storage/v2/" - export DATASET_BASE_URL="${AWS_BASE_URL}/api/dataset/v1/" - export LEGAL_BASE_URL="${AWS_BASE_URL}/api/legal/v1/" - export ENTITLEMENTS_BASE_URL="${AWS_BASE_URL}/api/entitlements/v2/" - export FILEDMS_BASE_URL="${AWS_BASE_URL}/api/dms/file/v1/" - export SEISMICSTORE_SVC_URL="${AWS_BASE_URL}/api/seismic-store/v3" - export WELLBORE_DDMS_URL="${AWS_BASE_URL}/api/os-wellbore-ddms" - export EDSDMS_BASE_URL="${AWS_BASE_URL}/api/dms/eds/v1/" - export SCHEMA_BASE_URL="${AWS_BASE_URL}/api/schema-service/v1/" - export ENTITLEMENT_V2_URL="${AWS_BASE_URL}/api/entitlements/v2/" - export LEGAL_QUEUE="https://sqs.${AWS_REGION}.amazonaws.com/${AWS_ACCOUNT_ID}/${RESOURCE_PREFIX}-legal-queue" - - export AWS_COGNITO_REGION="$AWS_REGION" - export AWS_S3_ENDPOINT="s3.${AWS_REGION}.amazonaws.com" - export AWS_S3_REGION="$AWS_REGION" - - export DYNAMO_DB_REGION="$AWS_REGION" - export DYNAMO_DB_ENDPOINT="dynamodb.${AWS_REGION}.amazonaws.com" - - export TABLE_PREFIX="$RESOURCE_PREFIX" - export AWS_CLUSTER_NAME="${RESOURCE_PREFIX}-core-cluster" - - echo "#### USING OSDU BASE URL: $AWS_BASE_URL ####" - -.aws_common_variables: - variables: - ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID - SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY - AWS_ACCESS_KEY_ID: $AWS_ACCOUNT_ACCESS_KEY_ID - AWS_SECRET_ACCESS_KEY: $AWS_ACCOUNT_SECRET_ACCESS_KEY - INTEGRATION_TEST_DIR: $AWS_TEST_SUBDIR - SERVICE_NAME: $AWS_SERVICE - BUILD_DIR: $AWS_BUILD_SUBDIR - ENVIRONMENT: $AWS_ENVIRONMENT - APPLICATION_NAME: os-$AWS_SERVICE - LOCAL_IMAGE_TAG: $CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG:$CI_COMMIT_SHA - AWS_IMAGE_TAG_BASE: ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_ECR_REGION}.amazonaws.com/os-${AWS_SERVICE} - OTHER_RELEVANT_DATA_COUNTRIES: $AWS_OTHER_RELEVANT_DATA_COUNTRIES - LEGAL_TAG : $AWS_LEGAL_TAG - TENANT_NAME : $AWS_TENANT_NAME - PRIVATE_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1 - PRIVATE_TENANT2: tenant2 - SHARED_TENANT: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2 - VENDOR: aws - DOMAIN: example.com - AWS_COGNITO_AUTH_FLOW: $AWS_COGNITO_AUTH_FLOW - AWS_COGNITO_AUTH_PARAMS_PASSWORD: $AWS_COGNITO_AUTH_PARAMS_PASSWORD - AWS_COGNITO_AUTH_PARAMS_USER: $AWS_COGNITO_AUTH_PARAMS_USER - AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS: $AWS_COGNITO_AUTH_PARAMS_USER_NO_ACCESS - AWS_ACCOUNT_ID: $AWS_ACCOUNT_ID - DEPLOY_ENV: empty - MY_TENANT: $AWS_TENANT_NAME - ENTITLEMENTS_TEST_TENANT: $AWS_ENTITLEMENTS_TEST_TENANT - LOG_LEVEL: INFO - SKIP_HTTP_TESTS: 'true' - DELIVERY_INT_TEST_BUCKET_NAME: $AWS_DELIVERY_INT_TEST_BUCKET_NAME - DEFAULT_DATA_PARTITION_ID_TENANT1: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT1 - DEFAULT_DATA_PARTITION_ID_TENANT2: $AWS_DEFAULT_DATA_PARTITION_ID_TENANT2 - #used by file->delivery int tests - DATA_PARTITION_ID: int-test-file - ENTITLEMENTS_DOMAIN: example.com - ELASTIC_PORT: 9200 - TEST_DAG_NAME: my_first_dag - SUBSCRIBER_SECRET: $AWS_SUBSCRIBER_SECRET - HMAC_SECRET: $AWS_HMAC_SECRET - PROVIDER_KEY: AWS_S3 - #File Service Variables - TIME_ZONE: UTC - USER_ID: $AWS_COGNITO_AUTH_PARAMS_USER - #Entitlements V2 variables--start-- - TENANT: opendes - SERVICE_PRINCIPAL_EMAIL: serviceprincipal@testing.com - #Entitlements V2 variables--end-- - -.aws: - tags: ['osdu-medium'] - image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-maven/aws-maven:v1.3 - environment: - name: AWS - extends: - - .aws_common_variables - - .aws_variables - before_script: - - mkdir -p ~/.aws - - | - cat > ~/.aws/credentials < Date: Mon, 6 Sep 2021 09:40:50 -0500 Subject: [PATCH 42/45] ci: fixing aws script --- .../devops/osdu/cloud-providers/aws-seismic-store-service.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 42a37263..7d4d58d8 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -60,6 +60,7 @@ aws-update-eks: stage: deploy needs: ['aws-containerize'] script: + - cd app/sdms - export KUBECONFIG=/tmp/kubeconfig-${RANDOM}.yaml - export EKS_CLUSTER_MGMT_ROLE=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterManagementRole'].OutputValue" --output text) - export EKS_CLUSTER_NAME=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterName'].OutputValue" --output text) @@ -68,7 +69,7 @@ aws-update-eks: #Some CLIs require a restrictive KUBECONFIG file - chmod 644 $KUBECONFIG - kubectl -n osdu-services rollout restart deployment/${AWS_EKS_DEPLOYMENT_NAME} - - kubectl -n osdu-services rollout status -w deployment/${AWS_EKS_DEPLOYMENT_NAME} --timeout=300s + - kubectl -n osdu-services rollout status -w deployment/${AWS_EKS_DEPLOYMENT_NAME} --timeout=900s only: variables: - $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true' && $AWS_DEPLOY_TARGET == 'EKS' -- GitLab From 202e88c327d798c08674c81cb6478a14d108eb78 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Mon, 6 Sep 2021 10:32:01 -0500 Subject: [PATCH 43/45] ci: fixing aws script --- .../devops/osdu/cloud-providers/aws-seismic-store-service.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index 7d4d58d8..a3c6f6a8 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -23,13 +23,14 @@ aws-containerize: stage: containerize needs: ['compile-and-unit-test'] script: - - cd app/sdms - | if [ -z $AWS_BUILDER_DOCKERFILE_PATH ] && [ -z $AWS_RUNTIME_DOCKERFILE_PATH ]; then echo Building BUILD_DIR/Dockerfile container docker build -f $BUILD_DIR/Dockerfile -t $LOCAL_IMAGE_TAG .; else echo Building BUILDER->RUNTIME containers; + cd app/sdms; + $PWD; docker build -t builder:latest --file $AWS_BUILDER_DOCKERFILE_PATH .; docker build -t $LOCAL_IMAGE_TAG --file $AWS_RUNTIME_DOCKERFILE_PATH --build-arg docker_builder_image=builder .; fi -- GitLab From 324b4943096379062b9c6bf2def0ee56dcc1cf37 Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Mon, 6 Sep 2021 10:38:03 -0500 Subject: [PATCH 44/45] ci: fixing aws script --- .../devops/osdu/cloud-providers/aws-seismic-store-service.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml index a3c6f6a8..d3dbc940 100644 --- a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -30,7 +30,6 @@ aws-containerize: else echo Building BUILDER->RUNTIME containers; cd app/sdms; - $PWD; docker build -t builder:latest --file $AWS_BUILDER_DOCKERFILE_PATH .; docker build -t $LOCAL_IMAGE_TAG --file $AWS_RUNTIME_DOCKERFILE_PATH --build-arg docker_builder_image=builder .; fi -- GitLab From ee03967fd3ff897b4bb74f094f76c4c61c4f2f3c Mon Sep 17 00:00:00 2001 From: Daniel Perez Date: Mon, 6 Sep 2021 11:06:11 -0500 Subject: [PATCH 45/45] ci: fixing aws script --- app/sdms/src/cloud/providers/aws/build-aws/runtime.Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/sdms/src/cloud/providers/aws/build-aws/runtime.Dockerfile b/app/sdms/src/cloud/providers/aws/build-aws/runtime.Dockerfile index b0fccd2d..ba9371c3 100644 --- a/app/sdms/src/cloud/providers/aws/build-aws/runtime.Dockerfile +++ b/app/sdms/src/cloud/providers/aws/build-aws/runtime.Dockerfile @@ -45,4 +45,6 @@ WORKDIR /seistore-service COPY src/cloud/providers/aws/build-aws/ssl.sh /seistore-service/ssl.sh COPY src/cloud/providers/aws/build-aws/entrypoint.sh /seistore-service/entrypoint.sh RUN npm ci --production +RUN chmod +x /seistore-service/ssl.sh +RUN chmod +x /seistore-service/entrypoint.sh ENTRYPOINT ["/bin/sh", "-c", "/seistore-service/entrypoint.sh"] \ No newline at end of file -- GitLab