Commit 894bf028 authored by Luc Yriarte's avatar Luc Yriarte
Browse files

Merge branch 'master' into local_integration_tests

parents 4094526a 86d31896
Pipeline #25683 passed with stages
in 8 minutes and 13 seconds
media
docs
variables:
PIP_REQUIREMENTS: "requirements.txt requirements_dev.txt"
OSDU_GCP_APPLICATION_NAME: wellbore-ddms
OSDU_GCP_SERVICE: wellbore-ddms
OSDU_GCP_ENV_VARS: CLOUD_PROVIDER=gcp,OS_WELLBORE_DDMS_DATA_PROJECT_ID=$OSDU_GCP_PROJECT,SERVICE_HOST_ENTITLEMENTS=$OSDU_GCP_ENTITLEMENTS_URL,SERVICE_HOST_SEARCH=$OSDU_GCP_SEARCH_HOST,SERVICE_HOST_STORAGE=$OSDU_GCP_STORAGE_URL
OSDU_GCP_CLOUD_RUN_PARAMETERS: "--port 8097"
OSDU_GCP_VENDOR: gcp
include:
- project: "osdu/platform/ci-cd-pipelines"
file: "standard-setup.yml"
......@@ -14,9 +20,11 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/gitlab-ultimate.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/osdu-gcp-cloudrun.yml'
# --------------------------------------------------------------------------------
deploy:
containerize:
extends:
- .skipForTriggeringMergeRequests
......@@ -47,3 +55,21 @@ deploy:
- echo ---- ---- PUSH IMAGE
- docker push $IMAGE_TAG
- docker push $CI_REGISTRY_IMAGE:latest
osdu-gcp-containerize:
script:
- gcloud auth activate-service-account --key-file $OSDU_GCP_DEPLOY_FILE
- gcloud config set project $OSDU_GCP_PROJECT
- touch .gcloudignore
- docker build -t gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_APPLICATION_NAME-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA --rm . -f ./build/Dockerfile
- gcloud docker -- push gcr.io/$OSDU_GCP_PROJECT/$OSDU_GCP_APPLICATION_NAME/$OSDU_GCP_APPLICATION_NAME-$OSDU_GCP_VENDOR:$CI_COMMIT_SHORT_SHA
cache: { }
tags: [ 'osdu-medium' ]
osdu-gcp-test-python:
allow_failure: true
script:
- echo "STUB"; exit 1
osdu-gcp-test:
extends: []
{
"python.pythonPath": "/usr/bin/python3"
}
\ No newline at end of file
......@@ -9,6 +9,8 @@ The following software have components provided under the terms of this license:
- aiohttp (from https://github.com/aio-libs/aiohttp/)
- async-timeout (from https://github.com/aio-libs/async_timeout/)
- boto3 (from https://github.com/boto/boto3)
- botocore (from https://github.com/boto/botocore)
- coverage (from https://coverage.readthedocs.io)
- cryptography (from https://github.com/pyca/cryptography)
- google-api-core (from https://github.com/GoogleCloudPlatform/google-cloud-python)
......@@ -37,6 +39,7 @@ The following software have components provided under the terms of this license:
- requests (from http://python-requests.org)
- rfc3986 (from https://rfc3986.readthedocs.org)
- rsa (from https://stuvel.eu/rsa)
- s3transfer (from https://github.com/boto/s3transfer)
- sniffio (from https://github.com/python-trio/sniffio)
- structlog (from http://www.structlog.org/)
- yarl (from https://github.com/aio-libs/yarl/)
......@@ -178,12 +181,14 @@ The following software have components provided under the terms of this license:
- azure-keyvault-keys (from https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/keyvault/azure-keyvault-keys)
- azure-keyvault-secrets (from https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/keyvault/azure-keyvault-secrets)
- azure-storage-blob (from https://github.com/Azure/azure-storage-python)
- botocore (from https://github.com/boto/botocore)
- cachetools (from https://github.com/tkem/cachetools)
- cffi (from http://cffi.readthedocs.org)
- coverage (from https://coverage.readthedocs.io)
- fastapi (from https://github.com/tiangolo/fastapi)
- h11 (from https://github.com/python-hyper/h11)
- iniconfig (from http://github.com/RonnyPfannschmidt/iniconfig)
- jmespath (from https://github.com/jmespath/jmespath.py)
- jsonschema (from http://github.com/Julian/jsonschema)
- mockito (from https://github.com/kaste/mockito-python)
- msal (from https://github.com/AzureAD/microsoft-authentication-library-for-python)
......@@ -254,6 +259,7 @@ public-domain
========================================================================
The following software have components provided under the terms of this license:
- botocore (from https://github.com/boto/botocore)
- coverage (from https://coverage.readthedocs.io)
- py (from http://pylib.readthedocs.org/)
- pytz (from http://pythonhosted.org/pytz)
......
......@@ -270,17 +270,14 @@ Then access app on `http://127.0.0.1:<LOCAL_PORT>/api/os-wellbore-ddms/docs`
A Personal Access Token (PAT) is required to pull all the python packages.
```bash
USER=<username>
PAT=<PAT>
# Set PIP_EXTRA_URL
PIP_EXTRA_URL="<URL>"
PIP_EXTRA_URL=https://community.opengroup.org/groups/osdu/platform/domain-data-mgmt-services/wellbore/-/packages
# Set IMAGE_TAG
IMAGE_TAG="os-wellbore-ddms:dev"
# Build Image
docker build -t=$IMAGE_TAG --rm . -f ./build/dockerfile --build-arg PIP_EXTRA_URL=$PIP_EXTRA_URL --build-arg PIP_WHEEL_DIR=python-packages
docker build -t=$IMAGE_TAG --rm . -f ./build/Dockerfile --build-arg PIP_EXTRA_URL=$PIP_EXTRA_URL --build-arg PIP_WHEEL_DIR=python-packages
```
#### Run Image
......
##################################################################
# Create a first image with credentials. It will download the dep.
##################################################################
FROM python:3.7-slim-buster as build
WORKDIR /root/build_directory
LABEL maintainer="OSDU Wellbore data domain services team <WellboreDDMS@slb.com>"
COPY requirements.txt .
RUN pip wheel uvicorn --wheel-dir=python-packages
ARG PIP_EXTRA_URL
ARG PIP_WHEEL_DIR
RUN pip wheel -r requirements.txt --extra-index-url=$PIP_EXTRA_URL --wheel-dir=$PIP_WHEEL_DIR
#################################################################
# Create a second image without credentials
#################################################################
FROM python:3.7-slim-buster
RUN apt-get update && apt-get install -y git
COPY requirements.txt requirements_dev.txt ./
COPY --from=build /root/build_directory .
RUN pip install --force-reinstall --ignore-installed --upgrade --no-index --no-deps python-packages/*
RUN pip install -r requirements.txt
RUN pip install -r requirements_dev.txt
COPY ./app /app
ENV PYTHONPATH=./
......@@ -46,4 +27,3 @@ RUN addgroup --system appuser && adduser --system appuser && adduser appuser app
USER appuser
CMD ["uvicorn", "app.wdms_app:wdms_app", "--host", "0.0.0.0", "--port", "8097"]
stages:
- stage: build
jobs:
- job: unit_test_version3_7
steps:
- template: steps-unit-tests.yaml
parameters:
python_version: '3.7'
- job: unit_test_version3_8
steps:
- template: steps-unit-tests.yaml
parameters:
python_version: '3.8'
include_coverage: false
- job: build
dependsOn:
- unit_test_version3_7
- unit_test_version3_8
steps:
- template: steps-build.yaml
parameters:
acr_names:
- ${{ variables.global_acr_name }}
- job: setupVariables
steps:
- task: Bash@3
displayName: 'setup basePath'
name: 'setupBasePath'
env:
build_reason: $(Build.Reason)
pull_request_id: $(System.PullRequest.PullRequestId)
inputs:
targetType: 'inline'
script: |
set -e
echo "build_reason: $build_reason"
echo "pull_request_id: $pull_request_id"
basePath=/api/os-wellbore-ddms
if [ "${build_reason}" == "PullRequest" ]; then
echo "Pull Request. Changing BasePath"
basePath=$basePath---pr${pull_request_id}
fi
echo "setting: basePath=$basePath"
echo "##vso[task.setvariable variable=basePath;isOutput=true]${basePath}"
\ No newline at end of file
parameters:
- name: providers
type: object
default: []
stages:
- ${{ each provider in parameters.providers }}:
- ${{ each environment in provider.environments }}:
- stage: 'Deploy_${{ provider.name }}_R3MVP_${{ environment }}'
variables:
- group: 'R3MVP - ${{ provider.name }} Target Env - ${{ environment }}'
- template: 'vars/env_vars.yml'
- name: env
value: ${{ environment }}
condition: "and(succeeded(), eq(variables['build.sourceBranch'], 'refs/heads/master'))"
jobs:
- deployment: Deploy
pool:
vmImage: "ubuntu-latest"
environment: ${{ environment }}
strategy:
runOnce:
deploy:
steps:
- template: steps-define-variables.yaml
- template: steps-push-image.yml
parameters:
azureSubscription: $(SERVICE_CONNECTION_NAME)
sourceACRName: $(us_acr_name)
destinationACRName: $(CONTAINER_REGISTRY_NAME)
environment: ${{ environment }}
imageTag: $(defineVariables.tag_name_output)
appName: $(app)
- template: devops/tasks/aks-deployment-steps.yml@pipelineTemplatesRepo
parameters:
environment: ${{ environment }}
serviceName: $(app)
providerName: ${{ provider.short_name }}
chartPath: $(chartPath)
valuesFile: $(valuesFile)
hldRegPath: $(hldRegPath)
- template: devops/tasks/flux-service-wait.yml@pipelineTemplatesRepo
parameters:
environment: ${{ environment }}
serviceName: $(fluxServiceName)
imageRepoName: $(app)
skipDeploy: false
- stage: Test_${{ provider.name }}_R3MVP_${{ environment }}
dependsOn:
- Deploy_${{ provider.name }}_R3MVP_${{ environment }}
variables:
- group: 'R3MVP - ${{ provider.name }} Target Env - ${{ environment }}'
- template: 'vars/env_vars.yml'
- template: 'vars/global_vars.yml'
condition: "and(succeeded(), eq(variables['build.sourceBranch'], 'refs/heads/master'))"
jobs:
- job: e2e_test
steps:
- template: steps-e2e-tests.yaml
parameters:
baseUrl: "https://$(DNS_HOST)"
basePath: ${{ variables.base_path }}
aclDomain: ${{ variables.acl_domain }}
legalTag: ${{ variables.legal_tag }}
dataPartition: ${{ variables.data_partition }}
cloudProvider: ${{ provider.short_name }}
testServiceAccount: TO_BE_DEFINED
parameters:
- name: acr_names
type: object #List of string
steps:
- template: steps-define-variables.yaml
- task: Bash@3
displayName: 'set version info'
inputs:
targetType: 'inline'
script: |
build_number=$(Build.BuildNumber)
patch_number=${build_number:0:8}
python ./build/set_version.py build_number=$build_number patch_number=$patch_number
- task: Bash@3
displayName: 'prepare artifacts'
inputs:
targetType: 'inline'
script: |
echo ----- GET IMAGE TAG ------------=
echo $(tag_name)
echo ----- PUBLISH ARTIFACTS ------------
echo $(tag_name)> $(Build.ArtifactStagingDirectory)/image_tag.txt
directories_to_copy=(deploy scripts tests)
for directory_to_copy in ${directories_to_copy[@]}; do
echo copying $directory_to_copy/
mkdir $(Build.ArtifactStagingDirectory)/$directory_to_copy
cp -R $directory_to_copy/* $(Build.ArtifactStagingDirectory)/$directory_to_copy/
done
- task: Bash@3
displayName: 'build image'
inputs:
targetType: 'inline'
script: |
current_utc_date=`date --utc`
echo $current_utc_date
echo ----- BUILD IMAGE ------------
docker build -t=wdms-osdu:$(tag_name) \
-t=wdms-osdu:latest \
--rm . -f ./build/Dockerfile \
--build-arg PIP_EXTRA_URL=$PIP_EXTRA_INDEX_URL \
--build-arg PIP_WHEEL_DIR=python-packages \
--build-arg build_date="$current_utc_date" \
--build-arg build_number="$(Build.BuildNumber)" \
--build-arg commit_id=$(commit_id) \
--build-arg build_origin="AzureDevops build/$(Build.DefinitionName)" \
--build-arg commit_branch=$(Build.SourceBranch)
# Current implementation pushes the images to a couple of 'central' ACR.
# They are different from the ACR mapped to a given environment
# The steps to publish to the environment specific ACR happens in a deploy stage, pulling the image from one those central ACR first.
#
# This could eventually be changed to an other apprach:
# - Push the built image as a ADO artifact, which will be downlaoded during the deploy stage
# - Or rebuild the image during the deploy stage
- ${{ each acr_name in parameters.acr_names }}:
- task: AzureCLI@2
displayName: 'push image to acr "${{ acr_name }}""'
inputs:
azureSubscription: TO_BE_DEFINED
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
set -e
az acr login --name ${{ acr_name }}
docker tag wdms-osdu:$(tag_name) ${{ acr_name }}.azurecr.io/os-wellbore-ddms:$(tag_name)
docker push ${{ acr_name }}.azurecr.io/os-wellbore-ddms:$(tag_name)
steps:
- task: Bash@3
displayName: 'define variables'
name: 'defineVariables'
inputs:
targetType: 'inline'
script: |
source_full_version=$(Build.SourceVersion)
commit_id=${source_full_version:0:7}
echo "##vso[task.setvariable variable=commit_id;]${commit_id}"
tag_name=$commit_id
echo "##vso[task.setvariable variable=tag_name]${tag_name}"
echo "##vso[task.setvariable variable=tag_name_output;isOutput=true]${tag_name}"
parameters:
- name: testSA
type: string
default: "" #Needed only for Google Test SA
- name: deAudience
type: string
default: "" #Needed only for Google Test SA
- name: baseUrl
type: string
- name: osduBaseUrl
type: string
default: "" # if empty will use baseUrl instead
- name: basePath
type: string
- name: cloudProvider
type: string
- name: aclDomain
type: string
- name: legalTag
type: string
- name: dataPartition
type: string
- name: testServiceAccount
type: object
- name: skipCertValidation
# Used in case of invalid certificate (self-signed,...)
type: boolean
default: false
steps:
- bash: |
set -e
echo "parameters.skipCertValidation: ${{ parameters.skipCertValidation }}"
echo "parameters.baseUrl: ${{ parameters.baseUrl }}"
echo "parameters.osduBaseUrl: ${{ parameters.osduBaseUrl }}"
echo "parameters.basePath: ${{ parameters.basePath }}"
echo "parameters.cloudProvider: ${{ parameters.cloudProvider }}"
echo "parameters.aclDomain: ${{ parameters.aclDomain }}"
echo "parameters.legalTag: ${{ parameters.legalTag }}"
echo "parameters.dataPartition: ${{ parameters.dataPartition }}"
echo "parameters.testServiceAccount.type: ${{ parameters.testServiceAccount.type }}"
displayName: 'Display Info'
- task: Bash@3
displayName: 'testReportBaseName'
inputs:
targetType: 'inline'
script: |
prefix=$(System.StageName)#$(System.JobAttempt)
echo "##vso[task.setvariable variable=testReportBaseName]${prefix}"
- task: UsePythonVersion@0
inputs:
versionSpec: '3.7'
addToPath: true
#############
# TO_BE_DEFINED
#------------
# In the current version, here should be a couple of tasks which goal is to generate a token to be used for integration tests.
# the 'testServiceAccount' parameter could be used to pass information on how to get the token (reference to secret, type of Identity Provider...)
#
# the token is to be added as a secure output variable:
# echo "##vso[task.setvariable variable=token;issecret=true]${token}"
#
# The following section for a GCP Service Account Token is given as example.
# A better approach could eventually be considered such as:
# - pass the token as parameter, and delegate to a higher task the responsibility to generate the token depending on the Service Account Type
# - use some kind of injection mechanism to include a template specific to the service account type
#
###########
### GCP Service Account based token
- ${{ if eq(parameters.testServiceAccount.type, 'gcp') }}:
- task: DownloadSecureFile@1
displayName: 'Download secret File : ${{parameters.testServiceAccount.secretFile}}'
name: secretFile
inputs:
secureFile: ${{ parameters.testServiceAccount.secretFile }}
- bash: |
set -e
gcloud auth activate-service-account --key-file $(secretFile.secureFilePath)
displayName: 'SETUP - Authenticate for GCP'
- task: Bash@3
displayName: 'Generate Token for GCP'
inputs:
targetType: 'inline'
script: |
token=$(gcloud auth print-identity-token --audiences=${{ parameters.deAudience }})
echo "##vso[task.setvariable variable=token;issecret=true]${token}"
### End of GCP Service Account based token
- bash: |
set -e
insecure_flag=""
checkCert="True"
# Depending on the value of 'skipCertValidation' we add the '--insecure' option to newman
# and update checkCert for pytest
skipCertValidation="${{ lower(parameters.skipCertValidation) }}"
if $skipCertValidation
then
echo "Adding --insecure flag to newman to disable cert validation."
insecure_flag="--insecure"
checkCert=""
fi
echo "##vso[task.setvariable variable=insecure_flag]${insecure_flag}"
echo "##vso[task.setvariable variable=checkCert]${checkCert}"
displayName: 'Create test variables'
- bash: |
set -e
osduBaseUrl=${{ parameters.osduBaseUrl }}
if [ -z "$osduBaseUrl" ]
then
osduBaseUrl=${{ parameters.baseUrl }}
fi
python tests/integration/gen_postman_env.py --token $(token) --base_url ${osduBaseUrl} --cloud_provider ${{ parameters.cloudProvider }} --acl_domain ${{ parameters.aclDomain }} --legal_tag ${{ parameters.legalTag }} --data_partition ${{ parameters.dataPartition }}
newman run tests/dependencies/core_dependencies_test.postman_collection.json -e generated/postman_environment.json -r junit,cli --reporter-junit-export dependencies_tests_report_$(testReportBaseName).xml --timeout-request 10000 $(insecure_flag)
displayName: 'Check healthiness of Core services'
condition: ne('${{ parameters.cloudProvider }}', 'gcp')
continueOnError: true
- bash: |
set -e
appUrl=${{ parameters.baseUrl }}${{ parameters.basePath }}
echo "Testing App on ${appUrl}"
python tests/integration/gen_postman_env.py --token $(token) --base_url ${appUrl} --cloud_provider ${{ parameters.cloudProvider }} --acl_domain ${{ parameters.aclDomain }} --legal_tag ${{ parameters.legalTag }} --data_partition ${{ parameters.dataPartition }}
pip install virtualenv
virtualenv venv37
source venv37/bin/activate
pip install -r ./requirements_dev.txt
pytest tests/integration/functional/ --junit-xml=$(testReportBaseName)_int_tests_report.xml -o junit_suite_name="wdms_integration_$(testReportBaseName)" --environment=generated/postman_environment.json $(insecure_flag) --retry-on-error=502
deactivate
displayName: 'RUN INTEGRATION-E2E'
- bash: |
set -e
pip install virtualenv
virtualenv venv37
source venv37/bin/activate
pip install -r ./requirements_dev.txt
appUrl=${{ parameters.baseUrl }}${{ parameters.basePath }}
echo "Testing App on ${appUrl}"
pytest tests/integration/security/test_auth.py --base_url ${appUrl} --check_cert "$(checkCert)" --token="$(token)" --junit-xml=$(testReportBaseName)_auth_tests_report.xml -o junit_suite_name="wdms_auth_$(testReportBaseName)"
deactivate
displayName: 'RUN INTEGRATION-AUTH'
- task: PublishTestResults@2
inputs:
testResultsFormat: 'JUnit'
testResultsFiles: '**/*tests_report*.xml'
condition: always()
parameters:
azureSubscription: ''
sourceACRName: ''
destinationACRName: ''
environment: ''
imageTag: ''
appName: ''
steps:
# Current implementation pulls the Image from a 'central' ACR first.
# It is an ACR to which the image was pushed during the build stage
#
# This could eventually be changed to an other apprach:
# - Push the built image as a ADO artifact, which will be downlaoded during the deploy stage
# - Or rebuild the image during the deploy stage
- task: AzureCLI@2
displayName: "Pull from WDMS Core ACR"
inputs:
azureSubscription: TO_BE_DEFINED
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
echo "az acr login -n ${{ parameters.sourceACRName }}"
az acr login -n ${{ parameters.sourceACRName }}
echo "docker pull ${{ parameters.sourceACRName}}.azurecr.io/os-wellbore-ddms:${{ parameters.imageTag }}"
docker pull ${{ parameters.sourceACRName}}.azurecr.io/os-wellbore-ddms:${{ parameters.imageTag }}
- task: AzureCLI@2
displayName: "Push to R3 MVP ${{ parameters.environment }} ACR"
inputs:
azureSubscription: ${{ parameters.azureSubscription }}
scriptType: bash
scriptLocation: inlineScript
inlineScript: |
set -e
echo "${{ parameters.azureSubscription }}"
echo "${{ parameters.destinationACRName }}"
az acr login --name ${{ parameters.destinationACRName }}
docker tag ${{ parameters.sourceACRName }}.azurecr.io/os-wellbore-ddms:${{ parameters.imageTag }} ${{ parameters.destinationACRName }}.azurecr.io/${{ parameters.appName }}-${{ parameters.environment }}:$(Build.SourceVersion)
docker push ${{ parameters.destinationACRName }}.azurecr.io/${{ parameters.appName }}-${{ parameters.environment }}:$(Build.SourceVersion)
az acr repository show-tags --name ${{ parameters.destinationACRName }} --repository ${{ parameters.appName }}-${{ parameters.environment }} --orderby time_desc --top 3
\ No newline at end of file
parameters:
- name: python_version
type: string
default: ''
- name: include_coverage
type: boolean
default: true
steps:
# Use Python version
# Use the specified version of Python from the tool cache, optionally adding it to the PATH
- task: UsePythonVersion@0
inputs:
versionSpec: ${{ parameters.python_version }}
addToPath: true
- task: Bash@3
displayName: 'run unit tests'
inputs:
targetType: 'inline'
script: |
echo ----- setup: installing dependencies --------