Notice update and Load test updates

.gitlab-ci.yml

@@ -23,7 +23,7 @@ variables:
   AZURE_DEPLOYMENTS_SCRIPTS_SUBDIR: deployment/scripts
   AZURE_CLIENT_ID: $AZURE_PRINCIPAL_ID
   AZURE_CLIENT_SECRET: $AZURE_PRINCIPAL_SECRET
-  DATA_PARTITION: opendes
+  AZURE_DATA_PARTITION: opendes
   AZURE_AD_APP_RESOURCE_ID: $AZURE_APP_ID
   AZURE_TEST_SUBDIR: tests/azure
   KEYVAULT_URI: $KEYVAULT_URI

NOTICE

@@ -110,7 +110,7 @@ The following software have components provided under the terms of this license:
 - charset-normalizer (from https://github.com/Ousret/charset_normalizer)
 - coverage (from https://github.com/nedbat/coveragepy)
 - exceptiongroup (from https://pypi.org/project/exceptiongroup/1.0.1/, https://pypi.org/project/exceptiongroup/1.1.3/)
-- fastapi (from https://pypi.org/project/fastapi/0.103.0/, https://pypi.org/project/fastapi/0.86.0/)
+- fastapi (from https://pypi.org/project/fastapi/0.103.1/, https://pypi.org/project/fastapi/0.86.0/)
 - h11
 - iniconfig (from http://github.com/RonnyPfannschmidt/iniconfig, https://pypi.org/project/iniconfig/2.0.0/)
 - jmespath (from https://github.com/jmespath/jmespath.py)

loadtest/README.md

@@ -5,13 +5,21 @@
 * TOKEN
 * SERVICE_URL
 
+## Optionally the test supports an environment variable:
+* TARGET_DATA_PARTITION - the DATA_PARTITION of where policy is running this is used in templates and for API path
+  This is intended to only be used when you are testing from one data_partition and the service is in another.
+  If not provided, DATA_PARTITION will be used.
+  Please note not all functions will work in policy if your path data_partition is different from the request header data_partition_id
+
 ## expected to run in gitlab pipeline
 loadtest.yml
 
 ## Tests currently in loadtest
 * /health
-* /policies/osdu/partition/<data_partition>/dataauthz.rego
+* /policies/osdu/partition/<target_data_partition>/dataauthz.rego
+* /policies/osdu/partition/<target_data_partition>/search2.rego
 * /evaluate
+* /translate
 
 ## Evaluate Notes:
 * /evaluate uses a template in ../frontend/admincli/tests/translate_data.json and

loadtest/locustfile.py

@@ -87,7 +87,7 @@ class PolicyServiceUser(HttpUser):
                 #print(r.json())
                 r.success()
             else:
-                r.failure(f"Failed: {r.status_code} {r.text}")
+                r.failure(f"Failed {policy_id}: {r.status_code} {r.text}")
 
     @tag('get_policies')
     # weight this task higher

loadtest/run-load-tests.sh

@@ -12,21 +12,23 @@ do
     esac
 done
 
+echo "#### run-load-tests ####"
+cat /etc/os-release
 python3 --version
 python3 -m venv env
 source env/bin/activate
+echo "#### Installing ####"
 python3 -m pip install --upgrade pip
 python3 -m pip install -r requirements.txt
 
+echo "#### Setting TOKEN ####"
 # AWS
 if [ ${CLOUD_PROVIDER} == "aws" ]; then
     export SERVICE_URL=$AWS_BASE_URL
     export TOKEN=$(python3 ../app/tests/aws/aws_jwt_client.py)
-    if [ -z $TARGET_DATA_PARTITION ]; then
-        export TARGET_DATA_PARTITION="osdu"
-    fi
     sleep 165
 elif [ ${CLOUD_PROVIDER} == "azure" ]; then
+    echo "## AZURE_DATA_PARTITION = $AZURE_DATA_PARTITION"
     python3 -m pip install -r requirements.txt
     export TOKEN=$(curl -ks -XPOST "https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/token" -d "grant_type=client_credentials&client_id=${AZURE_PRINCIPAL_ID}&client_secret=${AZURE_PRINCIPAL_SECRET}&resource=${AZURE_APP_ID}" | jq --raw-output '.access_token')
     export SERVICE_URL=https://${AZURE_DNS_NAME}
@@ -55,8 +57,20 @@ if [ -z $TOKEN ]; then
     exit 1
 fi
 
+if [ -z $DATA_PARTITION ]; then
+    echo "[ERROR] Not able to get data partition id (DATA_PARTITION)"
+    exit 1
+fi
+
+if [ -z $TARGET_DATA_PARTITION ]; then
+    export TARGET_DATA_PARTITION=$DATA_PARTITION
+fi
+
+echo "#### ENV ####"
 echo DATA_PARTITION: $DATA_PARTITION
 echo TARGET_DATA_PARTITION: $TARGET_DATA_PARTITION
 echo CLOUD_PROVIDER: $CLOUD_PROVIDER
 echo SERVICE_URL: $SERVICE_URL
+
+echo "#### Running ####"
 locust --headless --users 10 --spawn-rate 1 --run-time 10m -H $SERVICE_URL --only-summary --csv=stats_${CLOUD_PROVIDER}_