diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86c2e442162ca60ce99fc7e4ce4d501d4886befa..3bab8144dd3a70ce56341317d5126d90644ea808 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,117 +1,8 @@ -variables: - #seismic store service variables - ENVIRONMENT: cloud - ISGITLAB: "true" - PORT: 80 - REPLICA: 1 - UTEST_RUNTIME_IMAGE: seistore-svc-runtime - SDMS_MIN_REPLICAS: 1 - SDMS_MAX_REPLICAS: 5 - - #aws variables - AWS_SERVICE: seismic-store - AWS_BUILD_SUBDIR: src/cloud/providers/aws/build-aws - AWS_TEST_SUBDIR: tests - AWS_ENVIRONMENT: dev - AWS_DEPLOY_TARGET: EKS - AWS_EKS_DEPLOYMENT_NAME: os-seismic-store - AWS_BUILDER_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/builder.Dockerfile - AWS_RUNTIME_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/runtime.Dockerfile - # skipping tests here. Using a local file to run tests - AWS_SKIP_TESTS: "true" - #azure variables - AZURE_SERVICE: seismic-store-service - #gcp variables - GCP_APPLICATION_NAME: os-seismic-store-service - GCP_DEPLOY_FILE: $OSDU_GCP_DEPLOY_FILE - GCP_PROJECT: opendes-evt - GCP_SERVICE: seismic-store-service - GCP_VENDOR: gcp - #ibm variables - IBM_OPENSHIFT_PROJECT: og-cicd - IBM_VENDOR: ibm - - OSDU_GCP_LOG_LEVEL: INFO - OSDU_GCP_SERVICE: seismic-store - OSDU_GCP_VENDOR: gcp - OSDU_GCP_APPLICATION_NAME: seismic-store - OSDU_GCP_APPLICATION: seismic-store - OSDU_GCP_ENTITLEMENT_BASE_URL_PATH: /entitlements/v2 - OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY: data-partition-id - OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE: https://community.osdu-gcp.go3-nrg.projects.epam.com/api - OSDU_GCP_DES_SERVICE_HOST_STORAGE: https://os-storage-attcrcktoa-uc.a.run.app/api - OSDU_GCP_ENV_VARS: CLOUDPROVIDER=${OSDU_GCP_CLOUD_PROVIDER},DES_SERVICE_HOST_PARTITION=${OSDU_GCP_PARTITION_API},ENTITLEMENT_BASE_URL_PATH=${OSDU_GCP_ENTITLEMENT_BASE_URL_PATH},DATA_PARTITION_REST_HEADER_KEY=${OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY},DES_SERVICE_HOST_STORAGE=${OSDU_GCP_DES_SERVICE_HOST_STORAGE},DES_SERVICE_HOST_COMPLIANCE=${OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE},SEISTORE_DES_TARGET_AUDIENCE=${GOOGLE_AUDIENCE},SERVICE_CLOUD_PROJECT=${OSDU_GCP_PROJECT},APP_ENVIRONMENT_IDENTIFIER=${TENANT},IMP_SERVICE_ACCOUNT_SIGNER=${OSDU_GCP_IMP_SERVICE_ACCOUNT_SIGNER},DES_SERVICE_HOST_ENTITLEMENT=${OSDU_GCP_ENTITLEMENTS_V2_BASE_URL},SEISTORE_DES_APPKEY=${OSDU_GCP_SEISTORE_DES_APPKEY},DES_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_DES_REDIS_INSTANCE_ADDRESS},DES_REDIS_INSTANCE_PORT=${OSDU_GCP_DES_REDIS_INSTANCE_PORT},LOCKSMAP_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_LOCKSMAP_REDIS_INSTANCE_ADDRESS} --vpc-connector=$OSDU_GCP_VPC_CONNECTOR - include: # pipeline logic - project: "osdu/platform/ci-cd-pipelines" file: "standard-setup.yml" - # build - - local: "devops/osdu/build/seismic-store-service.yml" - - # scan - # fossa - - local: "devops/osdu/scanners/fossa-node.yml" - - # lint - - local: "/devops/osdu/scanners/lint-node.yml" - - # scan for secrets - - local: "/devops/osdu/scanners/scan-for-secrets-node.yml" - - # containerize - - project: "osdu/platform/ci-cd-pipelines" - file: "containerize/seismic-store-service.yml" - - # aws - - project: "osdu/platform/ci-cd-pipelines" - file: "cloud-providers/aws-global.yml" - - - local: "/devops/aws/awstest.yml" - - # deploy - #azure - - local: "/devops/osdu/cloud-providers/azure-seismic-store-service.yml" - - #ibm - - project: "osdu/platform/ci-cd-pipelines" - file: "cloud-providers/ibm-seismic-store-service.yml" - - #gcp - - project: "osdu/platform/ci-cd-pipelines" - ref: "master" - file: "cloud-providers/osdu-gcp-cloudrun.yml" - -osdu-gcp-test: - script: - - echo "Tests not implemented" - extends: - - .osdu-gcp-variables - only: - variables: - - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE != 'python' - -osdu-gcp-test-python: - script: - - echo "Tests not implemented" - only: - variables: - - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE == 'python' - -osdu-gcp-containerize-gitlab: - stage: containerize - needs: ["compile-and-unit-test"] - tags: ["osdu-medium"] - extends: .osdu-gcp-variables - image: docker:19.03 - cache: {} - allow_failure: true - script: - - export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:latest" ; fi - - docker build -t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --file docker/runtime.Dockerfile . - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY - - docker push $CI_REGISTRY_IMAGE/osdu-gcp - only: - variables: - - $OSDU_GCP == 'true' + # app + # sdms + - local: "app/sdms/.gitlab-ci.yml" \ No newline at end of file diff --git a/README.md b/README.md index c5ef000d117df5d4aeab7b876a1eb2161673bd48..71d4aecb46615aab910a3baf1ad738b7aab60dd2 100644 --- a/README.md +++ b/README.md @@ -7,48 +7,3 @@ Saving a dataset on a cloud-based storage, as single entity, may be a problem wh Seismic Store is a cloud-based solution composed by restful micro-services, client APIs and tools designed to implement a multi-object storage approach. The system saves objects that compose a dataset as a hierarchical data structure in a cloud storage and the dataset properties as a metadata entry in a no-relational catalogue. Having the datasets stored as multiple independent objects improve the overall performance, as generic I/O operations, for example read or write objects, can be easily parallelized. Seismic Store manages data authorization at service level by protecting access to storage bucket resources. Only service authorized users are enabled to directly access a storage resource. The service implements a mechanism that generates an “impersonation token” by authorizing long running/background production jobs to access data without requiring further user interactions. - -![service architecture diagram](docs/sdms-architecture-diagram-plain.png "Service Architecture Diagram") - -```python -# build the service -npm run build - -# start the service -npm run start - -# run integral/unit tests -npm run test - -# run integral/unit tests and generate the code coverage -npm run code-coverage - -# run the regression/e2e test suite -./tests/e2e/run_e2e_tests.sh \ - --seistore-svc-url="seismic store service url" \ - --seistore-svc-api-key="seismic store service api key" \ - --user-idtoken="user or service agent idtoken" \ - --tenant="seistore working tenant" \ - --subproject="seistore working subproject" \ - --admin-email="admin email" - -# run the parallel regression/e2e test suite (add the --run-parallel option) -./tests/e2e/run_e2e_tests.sh \ - --seistore-svc-url="seismic store service url" \ - --seistore-svc-api-key="seismic store service api key" \ - --user-idtoken="user or service agent idtoken" \ - --tenant="seistore working tenant" \ - --subproject="seistore working subproject" \ - --admin-email="admin email" \ - --run-parallel - -# run the linter on sources -tslint -c tslint.json -p tsconfig.json 'src/**/*.ts' -``` - -## Environment configuration - -Environment variables can be provided with a `.env` file in the root of the project to be consumed -by [dotenv](https://github.com/motdotla/dotenv). Environment variables are [preloaded](https://github.com/motdotla/dotenv#preload) -by the `npm start` command with the argument `-r dotenv/config`. A template `.env` file can be found -in `/docs/templates/.env-sample`. diff --git a/.devcontainer/Dockerfile b/app/sdms/.devcontainer/Dockerfile similarity index 100% rename from .devcontainer/Dockerfile rename to app/sdms/.devcontainer/Dockerfile diff --git a/.devcontainer/README.md b/app/sdms/.devcontainer/README.md similarity index 100% rename from .devcontainer/README.md rename to app/sdms/.devcontainer/README.md diff --git a/.devcontainer/bootstrap.sh b/app/sdms/.devcontainer/bootstrap.sh similarity index 100% rename from .devcontainer/bootstrap.sh rename to app/sdms/.devcontainer/bootstrap.sh diff --git a/.devcontainer/devcontainer.json b/app/sdms/.devcontainer/devcontainer.json similarity index 100% rename from .devcontainer/devcontainer.json rename to app/sdms/.devcontainer/devcontainer.json diff --git a/.devcontainer/readme/bash.png b/app/sdms/.devcontainer/readme/bash.png similarity index 100% rename from .devcontainer/readme/bash.png rename to app/sdms/.devcontainer/readme/bash.png diff --git a/.devcontainer/readme/command-palette.png b/app/sdms/.devcontainer/readme/command-palette.png similarity index 100% rename from .devcontainer/readme/command-palette.png rename to app/sdms/.devcontainer/readme/command-palette.png diff --git a/.devcontainer/readme/container.png b/app/sdms/.devcontainer/readme/container.png similarity index 100% rename from .devcontainer/readme/container.png rename to app/sdms/.devcontainer/readme/container.png diff --git a/.dockerignore b/app/sdms/.dockerignore similarity index 100% rename from .dockerignore rename to app/sdms/.dockerignore diff --git a/.fossa.yml b/app/sdms/.fossa.yml similarity index 100% rename from .fossa.yml rename to app/sdms/.fossa.yml diff --git a/.gitattributes b/app/sdms/.gitattributes similarity index 100% rename from .gitattributes rename to app/sdms/.gitattributes diff --git a/.gitignore b/app/sdms/.gitignore similarity index 100% rename from .gitignore rename to app/sdms/.gitignore diff --git a/app/sdms/.gitlab-ci.yml b/app/sdms/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..59342f9f3c7a7f5c306f3afa24813effdae5809f --- /dev/null +++ b/app/sdms/.gitlab-ci.yml @@ -0,0 +1,39 @@ +variables: + #seismic store service variables + ENVIRONMENT: cloud + ISGITLAB: "true" + PORT: 80 + REPLICA: 1 + UTEST_RUNTIME_IMAGE: seistore-svc-runtime + +include: + + # build + - local: "app/sdms/devops/osdu/build/seismic-store-service.yml" + + # scan + # fossa + - local: "app/sdms/devops/osdu/scanners/fossa-node.yml" + + # lint + - local: "app/sdms/devops/osdu/scanners/lint-node.yml" + + # scan for secrets + - local: "app/sdms/devops/osdu/scanners/scan-for-secrets-node.yml" + + # containerize + # osdu + - project: "app/sdms/devops/osdu/contenerize/seismic-store-service.yml" + + # deploy + # azure + - local: "app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml" + + # aws + - local: "app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml" + + # gcp + - local: "app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml" + + # ibm + - local: "app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml" \ No newline at end of file diff --git a/.husky/pre-commit b/app/sdms/.husky/pre-commit similarity index 100% rename from .husky/pre-commit rename to app/sdms/.husky/pre-commit diff --git a/.mocharc.json b/app/sdms/.mocharc.json similarity index 100% rename from .mocharc.json rename to app/sdms/.mocharc.json diff --git a/CHANGELOG.md b/app/sdms/CHANGELOG.md similarity index 100% rename from CHANGELOG.md rename to app/sdms/CHANGELOG.md diff --git a/LICENSE b/app/sdms/LICENSE similarity index 100% rename from LICENSE rename to app/sdms/LICENSE diff --git a/NOTICE b/app/sdms/NOTICE similarity index 100% rename from NOTICE rename to app/sdms/NOTICE diff --git a/app/sdms/README.md b/app/sdms/README.md new file mode 100644 index 0000000000000000000000000000000000000000..c5ef000d117df5d4aeab7b876a1eb2161673bd48 --- /dev/null +++ b/app/sdms/README.md @@ -0,0 +1,54 @@ +# SEISMIC STORE + +Seismic Store is a cloud-based solution designed to store and manage datasets of any size in the cloud by enabling a secure way to access them through a scoped authorization mechanism. Seismic Store overcomes the object size limitations imposed by a cloud provider, by managing generic datasets as multi independent objects and, therefore, provides a generic, reliable and a better performed solution to handle data on a cloud storage. + +Saving a dataset on a cloud-based storage, as single entity, may be a problem when it exceeds the maximum allowed object size. Adopting a single object storage approach is also not an optimal solution in terms of performance as a single entity cannot be easily uploaded and downloaded directly in parallel. + +Seismic Store is a cloud-based solution composed by restful micro-services, client APIs and tools designed to implement a multi-object storage approach. The system saves objects that compose a dataset as a hierarchical data structure in a cloud storage and the dataset properties as a metadata entry in a no-relational catalogue. Having the datasets stored as multiple independent objects improve the overall performance, as generic I/O operations, for example read or write objects, can be easily parallelized. + +Seismic Store manages data authorization at service level by protecting access to storage bucket resources. Only service authorized users are enabled to directly access a storage resource. The service implements a mechanism that generates an “impersonation token” by authorizing long running/background production jobs to access data without requiring further user interactions. + +![service architecture diagram](docs/sdms-architecture-diagram-plain.png "Service Architecture Diagram") + +```python +# build the service +npm run build + +# start the service +npm run start + +# run integral/unit tests +npm run test + +# run integral/unit tests and generate the code coverage +npm run code-coverage + +# run the regression/e2e test suite +./tests/e2e/run_e2e_tests.sh \ + --seistore-svc-url="seismic store service url" \ + --seistore-svc-api-key="seismic store service api key" \ + --user-idtoken="user or service agent idtoken" \ + --tenant="seistore working tenant" \ + --subproject="seistore working subproject" \ + --admin-email="admin email" + +# run the parallel regression/e2e test suite (add the --run-parallel option) +./tests/e2e/run_e2e_tests.sh \ + --seistore-svc-url="seismic store service url" \ + --seistore-svc-api-key="seismic store service api key" \ + --user-idtoken="user or service agent idtoken" \ + --tenant="seistore working tenant" \ + --subproject="seistore working subproject" \ + --admin-email="admin email" \ + --run-parallel + +# run the linter on sources +tslint -c tslint.json -p tsconfig.json 'src/**/*.ts' +``` + +## Environment configuration + +Environment variables can be provided with a `.env` file in the root of the project to be consumed +by [dotenv](https://github.com/motdotla/dotenv). Environment variables are [preloaded](https://github.com/motdotla/dotenv#preload) +by the `npm start` command with the argument `-r dotenv/config`. A template `.env` file can be found +in `/docs/templates/.env-sample`. diff --git a/devops/aws/requirements.txt b/app/sdms/devops/aws/requirements.txt similarity index 100% rename from devops/aws/requirements.txt rename to app/sdms/devops/aws/requirements.txt diff --git a/devops/azure/chart/Chart.yaml b/app/sdms/devops/azure/chart/Chart.yaml similarity index 100% rename from devops/azure/chart/Chart.yaml rename to app/sdms/devops/azure/chart/Chart.yaml diff --git a/devops/azure/chart/helm-config-rollback.yaml b/app/sdms/devops/azure/chart/helm-config-rollback.yaml similarity index 100% rename from devops/azure/chart/helm-config-rollback.yaml rename to app/sdms/devops/azure/chart/helm-config-rollback.yaml diff --git a/devops/azure/chart/helm-config.yaml b/app/sdms/devops/azure/chart/helm-config.yaml similarity index 100% rename from devops/azure/chart/helm-config.yaml rename to app/sdms/devops/azure/chart/helm-config.yaml diff --git a/devops/azure/chart/templates/azure-istio-auth-policy.yaml b/app/sdms/devops/azure/chart/templates/azure-istio-auth-policy.yaml similarity index 100% rename from devops/azure/chart/templates/azure-istio-auth-policy.yaml rename to app/sdms/devops/azure/chart/templates/azure-istio-auth-policy.yaml diff --git a/devops/azure/chart/templates/configmap.yaml b/app/sdms/devops/azure/chart/templates/configmap.yaml similarity index 100% rename from devops/azure/chart/templates/configmap.yaml rename to app/sdms/devops/azure/chart/templates/configmap.yaml diff --git a/devops/azure/chart/templates/deployment.yaml b/app/sdms/devops/azure/chart/templates/deployment.yaml similarity index 100% rename from devops/azure/chart/templates/deployment.yaml rename to app/sdms/devops/azure/chart/templates/deployment.yaml diff --git a/devops/azure/chart/templates/hpa.yaml b/app/sdms/devops/azure/chart/templates/hpa.yaml similarity index 100% rename from devops/azure/chart/templates/hpa.yaml rename to app/sdms/devops/azure/chart/templates/hpa.yaml diff --git a/devops/azure/chart/templates/service.yaml b/app/sdms/devops/azure/chart/templates/service.yaml similarity index 100% rename from devops/azure/chart/templates/service.yaml rename to app/sdms/devops/azure/chart/templates/service.yaml diff --git a/devops/azure/development-pipeline.yml b/app/sdms/devops/azure/development-pipeline.yml similarity index 100% rename from devops/azure/development-pipeline.yml rename to app/sdms/devops/azure/development-pipeline.yml diff --git a/devops/azure/pipeline.yml b/app/sdms/devops/azure/pipeline.yml similarity index 100% rename from devops/azure/pipeline.yml rename to app/sdms/devops/azure/pipeline.yml diff --git a/devops/azure/template/build-stage.yml b/app/sdms/devops/azure/template/build-stage.yml similarity index 100% rename from devops/azure/template/build-stage.yml rename to app/sdms/devops/azure/template/build-stage.yml diff --git a/devops/azure/template/deploy-stage.yml b/app/sdms/devops/azure/template/deploy-stage.yml similarity index 100% rename from devops/azure/template/deploy-stage.yml rename to app/sdms/devops/azure/template/deploy-stage.yml diff --git a/devops/azure/template/task/aks-deployment-steps.yml b/app/sdms/devops/azure/template/task/aks-deployment-steps.yml similarity index 100% rename from devops/azure/template/task/aks-deployment-steps.yml rename to app/sdms/devops/azure/template/task/aks-deployment-steps.yml diff --git a/devops/azure/template/task/build-and-unit-test.yml b/app/sdms/devops/azure/template/task/build-and-unit-test.yml similarity index 100% rename from devops/azure/template/task/build-and-unit-test.yml rename to app/sdms/devops/azure/template/task/build-and-unit-test.yml diff --git a/devops/azure/template/task/build-runtime-image.yml b/app/sdms/devops/azure/template/task/build-runtime-image.yml similarity index 100% rename from devops/azure/template/task/build-runtime-image.yml rename to app/sdms/devops/azure/template/task/build-runtime-image.yml diff --git a/devops/azure/template/task/build-steps.yml b/app/sdms/devops/azure/template/task/build-steps.yml similarity index 100% rename from devops/azure/template/task/build-steps.yml rename to app/sdms/devops/azure/template/task/build-steps.yml diff --git a/devops/azure/template/task/copy-code-coverage-report.yml b/app/sdms/devops/azure/template/task/copy-code-coverage-report.yml similarity index 100% rename from devops/azure/template/task/copy-code-coverage-report.yml rename to app/sdms/devops/azure/template/task/copy-code-coverage-report.yml diff --git a/devops/azure/template/task/deployment-helm-steps.yml b/app/sdms/devops/azure/template/task/deployment-helm-steps.yml similarity index 100% rename from devops/azure/template/task/deployment-helm-steps.yml rename to app/sdms/devops/azure/template/task/deployment-helm-steps.yml diff --git a/devops/azure/template/task/deployment-image-steps.yml b/app/sdms/devops/azure/template/task/deployment-image-steps.yml similarity index 100% rename from devops/azure/template/task/deployment-image-steps.yml rename to app/sdms/devops/azure/template/task/deployment-image-steps.yml diff --git a/devops/azure/template/task/download-build-artifact.yml b/app/sdms/devops/azure/template/task/download-build-artifact.yml similarity index 100% rename from devops/azure/template/task/download-build-artifact.yml rename to app/sdms/devops/azure/template/task/download-build-artifact.yml diff --git a/devops/azure/template/task/e2e-steps.yml b/app/sdms/devops/azure/template/task/e2e-steps.yml similarity index 100% rename from devops/azure/template/task/e2e-steps.yml rename to app/sdms/devops/azure/template/task/e2e-steps.yml diff --git a/devops/azure/template/task/e2e.yml b/app/sdms/devops/azure/template/task/e2e.yml similarity index 100% rename from devops/azure/template/task/e2e.yml rename to app/sdms/devops/azure/template/task/e2e.yml diff --git a/devops/azure/template/task/flux-service-wait.yml b/app/sdms/devops/azure/template/task/flux-service-wait.yml similarity index 100% rename from devops/azure/template/task/flux-service-wait.yml rename to app/sdms/devops/azure/template/task/flux-service-wait.yml diff --git a/devops/azure/template/task/generate-token.yml b/app/sdms/devops/azure/template/task/generate-token.yml similarity index 100% rename from devops/azure/template/task/generate-token.yml rename to app/sdms/devops/azure/template/task/generate-token.yml diff --git a/devops/azure/template/task/gitops.yml b/app/sdms/devops/azure/template/task/gitops.yml similarity index 100% rename from devops/azure/template/task/gitops.yml rename to app/sdms/devops/azure/template/task/gitops.yml diff --git a/devops/azure/template/task/helm-template.yml b/app/sdms/devops/azure/template/task/helm-template.yml similarity index 100% rename from devops/azure/template/task/helm-template.yml rename to app/sdms/devops/azure/template/task/helm-template.yml diff --git a/devops/azure/template/task/install-e2e-requirements.yml b/app/sdms/devops/azure/template/task/install-e2e-requirements.yml similarity index 100% rename from devops/azure/template/task/install-e2e-requirements.yml rename to app/sdms/devops/azure/template/task/install-e2e-requirements.yml diff --git a/devops/azure/template/task/kube.yml b/app/sdms/devops/azure/template/task/kube.yml similarity index 100% rename from devops/azure/template/task/kube.yml rename to app/sdms/devops/azure/template/task/kube.yml diff --git a/devops/azure/template/task/publish-build-artifact.yml b/app/sdms/devops/azure/template/task/publish-build-artifact.yml similarity index 100% rename from devops/azure/template/task/publish-build-artifact.yml rename to app/sdms/devops/azure/template/task/publish-build-artifact.yml diff --git a/devops/azure/template/task/publish-code-coverage.yml b/app/sdms/devops/azure/template/task/publish-code-coverage.yml similarity index 100% rename from devops/azure/template/task/publish-code-coverage.yml rename to app/sdms/devops/azure/template/task/publish-code-coverage.yml diff --git a/devops/azure/template/task/publish-unit-test-results.yml b/app/sdms/devops/azure/template/task/publish-unit-test-results.yml similarity index 100% rename from devops/azure/template/task/publish-unit-test-results.yml rename to app/sdms/devops/azure/template/task/publish-unit-test-results.yml diff --git a/devops/azure/template/task/push-runtime-image.yml b/app/sdms/devops/azure/template/task/push-runtime-image.yml similarity index 100% rename from devops/azure/template/task/push-runtime-image.yml rename to app/sdms/devops/azure/template/task/push-runtime-image.yml diff --git a/devops/azure/template/task/rollback-steps.yml b/app/sdms/devops/azure/template/task/rollback-steps.yml similarity index 100% rename from devops/azure/template/task/rollback-steps.yml rename to app/sdms/devops/azure/template/task/rollback-steps.yml diff --git a/devops/azure/template/task/rollback.yml b/app/sdms/devops/azure/template/task/rollback.yml similarity index 100% rename from devops/azure/template/task/rollback.yml rename to app/sdms/devops/azure/template/task/rollback.yml diff --git a/devops/azure/template/task/run-e2e.yml b/app/sdms/devops/azure/template/task/run-e2e.yml similarity index 100% rename from devops/azure/template/task/run-e2e.yml rename to app/sdms/devops/azure/template/task/run-e2e.yml diff --git a/devops/docker/detect_secrets/.secrets.baseline b/app/sdms/devops/docker/detect_secrets/.secrets.baseline similarity index 100% rename from devops/docker/detect_secrets/.secrets.baseline rename to app/sdms/devops/docker/detect_secrets/.secrets.baseline diff --git a/devops/docker/detect_secrets/Dockerfile b/app/sdms/devops/docker/detect_secrets/Dockerfile similarity index 100% rename from devops/docker/detect_secrets/Dockerfile rename to app/sdms/devops/docker/detect_secrets/Dockerfile diff --git a/devops/docker/detect_secrets/README.md b/app/sdms/devops/docker/detect_secrets/README.md similarity index 100% rename from devops/docker/detect_secrets/README.md rename to app/sdms/devops/docker/detect_secrets/README.md diff --git a/devops/osdu/build/seismic-store-service.yml b/app/sdms/devops/osdu/build/seismic-store-service.yml similarity index 66% rename from devops/osdu/build/seismic-store-service.yml rename to app/sdms/devops/osdu/build/seismic-store-service.yml index 9f0ca562f577d2032d72413eae84b7024520d4e1..40fec8ea07b9df399af55af9589ace25bb1afab2 100644 --- a/devops/osdu/build/seismic-store-service.yml +++ b/app/sdms/devops/osdu/build/seismic-store-service.yml @@ -5,10 +5,18 @@ compile-and-unit-test: artifacts: reports: junit: test-results.xml - cobertura: coverage/cobertura-coverage.xml + cobertura: app/sdms/coverage/cobertura-coverage.xml paths: - - coverage/ + - app/sdms/coverage/ expire_in: 15 days script: + - cd app/sdms - chmod +x devops/scripts/utest_build_run.sh - devops/scripts/utest_build_run.sh + only: + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests diff --git a/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml new file mode 100644 index 0000000000000000000000000000000000000000..d3dbc940bdc69468521156ca54bddd3856907622 --- /dev/null +++ b/app/sdms/devops/osdu/cloud-providers/aws-seismic-store-service.yml @@ -0,0 +1,111 @@ +include: + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/aws-global.yml" + +variables: + #aws variables + AWS_SERVICE: seismic-store + AWS_BUILD_SUBDIR: src/cloud/providers/aws/build-aws + AWS_TEST_SUBDIR: tests + AWS_ENVIRONMENT: dev + AWS_DEPLOY_TARGET: EKS + AWS_EKS_DEPLOYMENT_NAME: os-seismic-store + AWS_BUILDER_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/builder.Dockerfile + AWS_RUNTIME_DOCKERFILE_PATH: src/cloud/providers/aws/build-aws/runtime.Dockerfile + # skipping tests here. Using a local file to run tests + AWS_SKIP_TESTS: "true" + +aws-containerize: + extends: + - .aws + - .aws_common_variables + - .aws_variables + stage: containerize + needs: ['compile-and-unit-test'] + script: + - | + if [ -z $AWS_BUILDER_DOCKERFILE_PATH ] && [ -z $AWS_RUNTIME_DOCKERFILE_PATH ]; then + echo Building BUILD_DIR/Dockerfile container + docker build -f $BUILD_DIR/Dockerfile -t $LOCAL_IMAGE_TAG .; + else + echo Building BUILDER->RUNTIME containers; + cd app/sdms; + docker build -t builder:latest --file $AWS_BUILDER_DOCKERFILE_PATH .; + docker build -t $LOCAL_IMAGE_TAG --file $AWS_RUNTIME_DOCKERFILE_PATH --build-arg docker_builder_image=builder .; + fi + # Push to the local container registry + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker push $LOCAL_IMAGE_TAG + # Push to Amazon's container registry + - $(aws ecr get-login --no-include-email --region $AWS_ECR_REGION) + - docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA + - docker tag $LOCAL_IMAGE_TAG $AWS_IMAGE_TAG_BASE:latest + - docker push $AWS_IMAGE_TAG_BASE:$CI_COMMIT_SHA + - docker push $AWS_IMAGE_TAG_BASE:latest + only: + variables: + - $AWS == 'true' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests + +aws-update-eks: + extends: + - .aws + - .aws_variables + - .aws_common_variables + stage: deploy + needs: ['aws-containerize'] + script: + - cd app/sdms + - export KUBECONFIG=/tmp/kubeconfig-${RANDOM}.yaml + - export EKS_CLUSTER_MGMT_ROLE=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterManagementRole'].OutputValue" --output text) + - export EKS_CLUSTER_NAME=$(aws cloudformation describe-stacks --region $AWS_REGION --stack-name $AWS_INFRA_STACK_NAME --query "Stacks[0].Outputs[?OutputKey=='MainEKSClusterName'].OutputValue" --output text) + - echo Using Role -- $EKS_CLUSTER_MGMT_ROLE + - aws eks update-kubeconfig --region $AWS_REGION --name $EKS_CLUSTER_NAME --role-arn $EKS_CLUSTER_MGMT_ROLE + #Some CLIs require a restrictive KUBECONFIG file + - chmod 644 $KUBECONFIG + - kubectl -n osdu-services rollout restart deployment/${AWS_EKS_DEPLOYMENT_NAME} + - kubectl -n osdu-services rollout status -w deployment/${AWS_EKS_DEPLOYMENT_NAME} --timeout=900s + only: + variables: + - $AWS_SKIP_DEPLOY != 'true' && $AWS == 'true' && $AWS_DEPLOY_TARGET == 'EKS' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests + +aws-test-newman: + extends: + - .aws + - .aws_common_variables + - .aws_variables + stage: integration + image: $CI_REGISTRY/osdu/platform/deployment-and-operations/base-containers-aws/aws-node/aws-node:v1.0-node14 + needs: [{ job: 'aws-update-ecs', optional: true }, { job: 'aws-update-eks', optional: true }] + script: + - cd app/sdms + - pip3 install -r devops/aws/requirements.txt + - svctoken=$(python3 devops/scripts/aws_jwt_client.py) + - npm install -g newman + - chmod +x ./tests/e2e/run_e2e_tests.sh + - ./tests/e2e/run_e2e_tests.sh --seistore-svc-url=${SEISMICSTORE_SVC_URL} --seistore-svc-api-key="NA" --user-idtoken="$svctoken" --user-idtoken="$svctoken" --tenant=opendes --subproject=awsdemosubproject --admin-email="${AWS_COGNITO_AUTH_PARAMS_USER}" --datapartition=opendes --legaltag01=opendes-sdmstestlegaltag --legaltag02=opendes-sdmstestlegaltag --subproject-long-name=subprojectlonggggggggggggggggggggggname --VCS-Provider="${ISGITLAB}" + only: + variables: + - $AWS == 'true' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests + artifacts: + when: always + paths: + - $INTEGRATION_TEST_DIR + expire_in: 14 days \ No newline at end of file diff --git a/devops/osdu/cloud-providers/azure-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml similarity index 72% rename from devops/osdu/cloud-providers/azure-seismic-store-service.yml rename to app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml index 48d96668af58a2fc28f1131385dadca1a83d13aa..1acfbf6f79615faa7b5df426f40a7a8316412ee9 100644 --- a/devops/osdu/cloud-providers/azure-seismic-store-service.yml +++ b/app/sdms/devops/osdu/cloud-providers/azure-seismic-store-service.yml @@ -1,4 +1,6 @@ variables: + #azure variables + AZURE_SERVICE: seismic-store-service CHART_PATH: devops/azure/chart CLOUD_PROVIDER: azure E2E_ADMIN_EMAIL: integration.test@azureglobal1.onmicrosoft.com @@ -10,8 +12,38 @@ variables: E2E_TENANT: opendes POD_IDENTITY: osdu-identity REDIS_INSTANCE_PORT: 6380 + SDMS_MIN_REPLICAS: 1 + SDMS_MAX_REPLICAS: 5 + +push_runtime_image_azure: + image: danielscholl/azure-build-image + tags: ["osdu-medium"] + stage: containerize + variables: + SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} + LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest + before_script: + - az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID + - az acr login -n $AZURE_REGISTRY + script: + # Azure Container Registry + - cd app/sdms + - docker build -t builder_sdms:latest --file docker/builder.Dockerfile . + - docker build -t $UTEST_RUNTIME_IMAGE --file docker/runtime.Dockerfile --build-arg docker_builder_image=builder_sdms . + - docker tag $UTEST_RUNTIME_IMAGE ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE + - docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE + - docker tag $UTEST_RUNTIME_IMAGE ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE + - docker push ${AZURE_REGISTRY}.azurecr.io/$LATEST_IMAGE + only: + variables: + - $AZURE == 'true' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests - azure_deploy: tags: ["osdu-medium"] image: danielscholl/azure-build-image @@ -28,6 +60,7 @@ azure_deploy: - az aks get-credentials -g $AZURE_UNIQUE-rg -n $AZURE_UNIQUE-aks script: # Replace values in config file + - cd app/sdms - cp ${CHART_PATH}/helm-config.yaml ${CHART_PATH}/values.yaml - sed -i 's,#{CONTAINER_REGISTRY_NAME}#,'$IMAGE',' ${CHART_PATH}/values.yaml - sed -i 's,#{DNS_HOST}#,'$DES_URL',' ${CHART_PATH}/values.yaml @@ -52,6 +85,12 @@ azure_deploy: only: variables: - $AZURE == 'true' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests azure_test: tags: ["osdu-medium"] @@ -64,6 +103,7 @@ azure_test: AZURE_TESTER_SERVICEPRINCIPAL_SECRET: $AZURE_PRINCIPAL_SECRET INTEGRATION_TESTER: $AZURE_PRINCIPAL_ID script: + - cd app/sdms - apt-get update - apt-get install -y python - apt-get install -y python-pip @@ -76,3 +116,9 @@ azure_test: only: variables: - $AZURE == 'true' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests \ No newline at end of file diff --git a/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml new file mode 100644 index 0000000000000000000000000000000000000000..157b8655ff552114b377fd4c2df87707c4c33f25 --- /dev/null +++ b/app/sdms/devops/osdu/cloud-providers/gcp-seismic-store-service.yml @@ -0,0 +1,183 @@ +# Copyright 2020 Google LLC +# Copyright 2017-2019, Schlumberger +# Copyright 2020 EPAM +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Global Protected Variables: +# +# OSDU_GCP +# OSDU_GCP_DEPLOY_FILE +# OSDU_GCP_SERVICE_ACCOUNT +# OSDU_GCP_INTEGRATION_TESTER +# OSDU_GCP_NO_DATA_ACCESS_TESTER + +include: + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/osdu-gcp-cloudrun.yml" + +variables: + #gcp variables + GCP_APPLICATION_NAME: os-seismic-store-service + GCP_DEPLOY_FILE: $OSDU_GCP_DEPLOY_FILE + GCP_PROJECT: opendes-evt + GCP_SERVICE: seismic-store-service + GCP_VENDOR: gcp + OSDU_GCP_LOG_LEVEL: INFO + OSDU_GCP_SERVICE: seismic-store + OSDU_GCP_VENDOR: gcp + OSDU_GCP_APPLICATION_NAME: seismic-store + OSDU_GCP_APPLICATION: seismic-store + OSDU_GCP_ENTITLEMENT_BASE_URL_PATH: /entitlements/v2 + OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY: data-partition-id + OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE: https://community.osdu-gcp.go3-nrg.projects.epam.com/api + OSDU_GCP_DES_SERVICE_HOST_STORAGE: https://os-storage-attcrcktoa-uc.a.run.app/api + OSDU_GCP_ENV_VARS: CLOUDPROVIDER=${OSDU_GCP_CLOUD_PROVIDER},DES_SERVICE_HOST_PARTITION=${OSDU_GCP_PARTITION_API},ENTITLEMENT_BASE_URL_PATH=${OSDU_GCP_ENTITLEMENT_BASE_URL_PATH},DATA_PARTITION_REST_HEADER_KEY=${OSDU_GCP_DATA_PARTITION_REST_HEADER_KEY},DES_SERVICE_HOST_STORAGE=${OSDU_GCP_DES_SERVICE_HOST_STORAGE},DES_SERVICE_HOST_COMPLIANCE=${OSDU_GCP_DES_SERVICE_HOST_COMPLIANCE},SEISTORE_DES_TARGET_AUDIENCE=${GOOGLE_AUDIENCE},SERVICE_CLOUD_PROJECT=${OSDU_GCP_PROJECT},APP_ENVIRONMENT_IDENTIFIER=${TENANT},IMP_SERVICE_ACCOUNT_SIGNER=${OSDU_GCP_IMP_SERVICE_ACCOUNT_SIGNER},DES_SERVICE_HOST_ENTITLEMENT=${OSDU_GCP_ENTITLEMENTS_V2_BASE_URL},SEISTORE_DES_APPKEY=${OSDU_GCP_SEISTORE_DES_APPKEY},DES_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_DES_REDIS_INSTANCE_ADDRESS},DES_REDIS_INSTANCE_PORT=${OSDU_GCP_DES_REDIS_INSTANCE_PORT},LOCKSMAP_REDIS_INSTANCE_ADDRESS=${OSDU_GCP_LOCKSMAP_REDIS_INSTANCE_ADDRESS} --vpc-connector=$OSDU_GCP_VPC_CONNECTOR + +push_runtime_image_gcp: + image: gcr.io/google.com/cloudsdktool/cloud-sdk + tags: ["osdu-medium"] + stage: containerize + variables: + SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} + LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest + before_script: + - gcloud auth activate-service-account --key-file="$GCP_DEPLOY_FILE" + - gcloud config set project $GCP_PROJECT + script: + # GCP Container Registry + - cd app/sdms + - gcloud builds submit --config provider/$GCP_VENDOR/cloudbuild/cloudbuild.yaml --substitutions=_GCP_SERVICE=$GCP_SERVICE,_APPLICATION_NAME=$GCP_APPLICATION_NAME,_PROVIDER_NAME=$GCP_VENDOR,_SHORT_SHA=$CI_COMMIT_SHORT_SHA,_PORT=$PORT + only: + variables: + - $GCP == 'true' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests + +osdu-gcp-containerize-gcloud: + stage: containerize + needs: ["compile-and-unit-test"] + extends: .osdu-gcp-variables + image: gcr.io/google.com/cloudsdktool/cloud-sdk + cache: {} + script: + - cd app/sdms + - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE + - gcloud config set project $OSDU_GCP_PROJECT + - touch .gcloudignore + - gcloud builds submit --config provider/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR/cloudbuild/cloudbuild.yaml --substitutions=_GCP_SERVICE=$OSDU_GCP_SERVICE,_APPLICATION_NAME=$OSDU_GCP_APPLICATION_NAME,_PROVIDER_NAME=$OSDU_GCP_VENDOR,_SHORT_SHA=$CI_COMMIT_SHORT_SHA,_PORT=$OSDU_GCP_PORT + only: + variables: + - $OSDU_GCP == 'true' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests + +osdu-gcp-containerize-gitlab: + stage: containerize + needs: ["compile-and-unit-test"] + tags: ["osdu-medium"] + extends: .osdu-gcp-variables + image: docker:19.03 + cache: {} + allow_failure: true + script: + - cd app/sdms + - export EXTRA_DOCKER_TAG=""; if [ "$CI_COMMIT_TAG" != "" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_TAG" ; elif [ "$CI_COMMIT_REF_NAME" = "master" ] ; then EXTRA_DOCKER_TAG="-t $CI_REGISTRY_IMAGE/osdu-gcp:latest" ; fi + - docker build -t $CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA $EXTRA_DOCKER_TAG --file docker/runtime.Dockerfile . + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker push $CI_REGISTRY_IMAGE/osdu-gcp + only: + variables: + - $OSDU_GCP == 'true' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests + +osdu-gcp-deploy: + image: gcr.io/google.com/cloudsdktool/cloud-sdk + needs: ["osdu-gcp-containerize-gcloud"] + stage: deploy + extends: .osdu-gcp-variables + cache: {} + script: + - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE + - gcloud config set project $OSDU_GCP_PROJECT + - > + gcloud beta run deploy $OSDU_GCP_APPLICATION_NAME + --image gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA + --platform managed + --region $OSDU_GCP_CLOUDRUN_REGION + --allow-unauthenticated + --service-account $OSDU_GCP_SERVICE_ACCOUNT + --memory 512M + $OSDU_GCP_CLOUD_RUN_PARAMETERS + --set-env-vars=LOG_LEVEL=${OSDU_GCP_LOG_LEVEL:="INFO"},$OSDU_GCP_ENV_VARS + - gcloud run services update-traffic $OSDU_GCP_APPLICATION_NAME --platform managed --region $OSDU_GCP_CLOUDRUN_REGION --to-latest + only: + variables: + - $OSDU_GCP == 'true' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests + +osdu-gcp-tagging: + image: google/cloud-sdk + needs: ["osdu-gcp-deploy"] + stage: integration + extends: .osdu-gcp-variables + cache: {} + script: + - gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE + - gcloud config set project $OSDU_GCP_PROJECT + - gcloud container images add-tag gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_SERVICE-$OSDU_GCP_VENDOR:$CI_COMMIT_TAG + only: + - tags + +osdu-gcp-test: + script: + - echo "Tests not implemented" + extends: + - .osdu-gcp-variables + only: + variables: + - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE != 'python' + refs: + - branches + - main + - merge_requests + +osdu-gcp-test-python: + script: + - echo "Tests not implemented" + only: + variables: + - $OSDU_GCP == 'true' && $OSDU_GCP_INT_TEST_TYPE == 'python' + refs: + - branches + - main + - merge_requests + - merge_requests \ No newline at end of file diff --git a/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml b/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml new file mode 100644 index 0000000000000000000000000000000000000000..fb1c5eb310f5ff9b4fdce5712fb3f7f718d811b2 --- /dev/null +++ b/app/sdms/devops/osdu/cloud-providers/ibm-seismic-store-service.yml @@ -0,0 +1,89 @@ +include: + - project: "osdu/platform/ci-cd-pipelines" + file: "cloud-providers/ibm-seismic-store-service.yml" + +variables: + #ibm variables + IBM_OPENSHIFT_PROJECT: og-cicd + IBM_VENDOR: ibm + +ibm-deploy: + extends: .ibm + stage: deploy + needs: ['compile-and-unit-test'] + script: + - cd app/sdms + - echo "In IBM deploy" + - oc project $IBM_OPENSHIFT_PROJECT + - oc get bc/$CI_PROJECT_NAME 2> /dev/null || oc new-build --name $CI_PROJECT_NAME --binary --strategy source --image-stream openshift/ubi8-nodejs-14 + - oc start-build $CI_PROJECT_NAME --from-dir=. --follow + - oc get service $CI_PROJECT_NAME 2> /dev/null || oc new-app $CI_PROJECT_NAME + only: + variables: + - $IBM == 'true' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests + +ibm-deploy-devpri: + extends: .ibm + stage: deploy + needs: ['compile-and-unit-test'] + script: + - cd app/sdms + - echo "In IBM deploy" + - oc login --token=$IBM_PRIMARY_DEV_TOKEN --server=$IBM_PRIMARY_DEV_URL + - oc project $IBM_OPENSHIFT_DEVPRI_PROJECT + - oc get bc/$CI_PROJECT_NAME 2> /dev/null || oc new-build --name $CI_PROJECT_NAME --binary --strategy source --image-stream openshift/ubi8-nodejs-14 + - oc start-build $CI_PROJECT_NAME --from-dir=. --follow + - oc get service $CI_PROJECT_NAME 2> /dev/null || oc new-app $CI_PROJECT_NAME + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + changes: + - app/sdms/**/* + +ibm-test: + tags: ["osdu-medium"] + image: node + stage: integration + extends: + - .ibm_variables + allow_failure: false + needs: ["ibm-deploy"] + script: + - cd app/sdms + - apt-get update + - apt-get install -y python + - apt-get install -y python-pip + - pip install requests + - svctoken=$(python devops/scripts/ibm_jwt_client.py) + - echo $svctoken + - npm install -g newman + - chmod +x ./tests/e2e/run_e2e_tests.sh + - ./tests/e2e/run_e2e_tests.sh --seistore-svc-url="https://${SEISMIC_HOST}/api/v3" --seistore-svc-api-key="NA" --user-idtoken="$svctoken" --user1-idtoken="$svctoken" --tenant="${IBM_SEISMIC_E2E_TENANT}" --subproject="${IBM_SEISMIC_E2E_SUBPROJECT}" --admin-email="${IBM_SEISMIC_E2E_ADMIN_EMAIL}" --datapartition="${IBM_SEISMIC_E2E_DATA_PARTITION}" --legaltag01="${IBM_SEISMIC_E2E_LEGALTAG01}" --legaltag02="${IBM_SEISMIC_E2E_LEGALTAG02}" --subproject-long-name="${IBM_SEISMIC_E2E_SUBPROJECT_LONG_NAME}" --newuser="${IBM_SEISMIC_E2E_NEWUSER}" --VCS-Provider="gitlab" + only: + variables: + - $IBM == 'true' + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests + +ibm-preship-promote: + extends: .ibm + stage: publish + needs: ['ibm-test'] + script: + - cd app/sdms + - oc login --token=$IBM_PRESHIP_OPENSHIFT_TOKEN --server=$IBM_PRESHIP_OPENSHIFT_URL + - oc project $IBM_OPENSHIFT_PRESHIP_PROJECT + - oc start-build og-$CI_PROJECT_NAME-ibm + rules: + - if: '$CI_COMMIT_BRANCH == "master"' + changes: + - app/sdms/**/* diff --git a/app/sdms/devops/osdu/contenerize/seismic-store-service.yml b/app/sdms/devops/osdu/contenerize/seismic-store-service.yml new file mode 100644 index 0000000000000000000000000000000000000000..934f3470af90c8a22507346ada07cf1b848af83e --- /dev/null +++ b/app/sdms/devops/osdu/contenerize/seismic-store-service.yml @@ -0,0 +1,27 @@ +push_runtime_image: + image: docker:19.03.12 + services: + - docker:19.03.12-dind + tags: ["osdu-medium"] + stage: containerize + variables: + SHA_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:${CI_COMMIT_SHA} + LATEST_IMAGE: ${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}:latest + before_script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + script: + # Gitlab Container Registry + - cd app/sdms + - docker build -t builder_sdms:latest --file docker/builder.Dockerfile . + - docker build -t $UTEST_RUNTIME_IMAGE --file docker/runtime.Dockerfile --build-arg docker_builder_image=builder_sdms . + - docker tag $UTEST_RUNTIME_IMAGE $CI_REGISTRY_IMAGE/$SHA_IMAGE + - docker push $CI_REGISTRY_IMAGE/$SHA_IMAGE + - docker tag $UTEST_RUNTIME_IMAGE $CI_REGISTRY_IMAGE/$LATEST_IMAGE + - docker push $CI_REGISTRY_IMAGE/$LATEST_IMAGE + only: + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests \ No newline at end of file diff --git a/app/sdms/devops/osdu/scanners/fossa-global.yml b/app/sdms/devops/osdu/scanners/fossa-global.yml new file mode 100644 index 0000000000000000000000000000000000000000..55472fabe02d4a1d541a43fbb15e67851b29eafe --- /dev/null +++ b/app/sdms/devops/osdu/scanners/fossa-global.yml @@ -0,0 +1,148 @@ +# This checks to see if the NOTICE file generated based on the current code +# matches the one that is committed in the repository. Because FOSSA can +# occassionally order the lines different between executions, we use a python +# program to parse the file into sections and sort each section. Then we compare +# and expect the same number of sections, names of sections, and contents +# (without consider order). + +include: + - project: "osdu/platform/ci-cd-pipelines" + file: "scanners/fossa-global.yml" + +fossa-check-notice: + image: $CI_REGISTRY/divido/fossa-cli/fossa-cli:v6.1 + stage: attribution + needs: ['fossa-analyze'] + tags: ['osdu-small'] + only: + variables: + - $FOSSA_API_KEY + changes: + - app/sdms/**/* + refs: + - branches + - main + - merge_requests + artifacts: + when: on_failure + paths: + - app/sdms/public + + script: + # fossa-check-for-licensing-issues needs a CI_COMMIT_BRANCH defined to know how to parse the FOSSA API results + # When building tags, this isn't defined by GitLab. In that case, we use the tag name instead. If that's not defined + # then things will fail and we'll have to make this smarter + - cd app/sdms + - test -z "$CI_COMMIT_BRANCH" && export CI_COMMIT_BRANCH="$CI_COMMIT_TAG" + + - mkdir -p public + - fossa report licenses --project "${CI_PROJECT_TITLE}" --branch "${CI_COMMIT_BRANCH}" > public/NOTICE + - mkdir committed-NOTICE generated-NOTICE + - | + python <