diff --git a/app/Makefile b/app/Makefile
index dd0cf41d062b1e75b5a65919519f3babe4730f54..c692212e12ab2fc6ffafda89958fc7b416c6082e 100644
--- a/app/Makefile
+++ b/app/Makefile
@@ -445,10 +445,13 @@ local_opa_ttab:
opa:
docker run \
+ -v ${PWD}:/conf \
-e ENTITLEMENTS_BASE_URL=${BASE_URL} \
-e LEGAL_BASE_URL=${BASE_URL} \
--name $(OPA_NAME) -p 8181:8181 openpolicyagent/opa \
+ --config-file=tests/opa.yaml \
run --server
+
opa2:
docker run -it --rm -p 8181:8181 -v ${PWD}:/conf openpolicyagent/opa run --server --config-file=tests/gsp_init.yaml
diff --git a/app/api/config_api.py b/app/api/config_api.py
index 0d3eb26e7536f4fdf70b75cd28b00e94301d9d8d..b08f63ba576986e3bc8f62403c5be00b09208c3f 100644
--- a/app/api/config_api.py
+++ b/app/api/config_api.py
@@ -72,7 +72,7 @@ def show_policy_config_details(
configmap = k8s.get_opa_config(namespace=conf.NAMESPACE, name=conf.OPA_CONFIG_MAP)
try:
- status_url = conf.OPA_STATUS_API
+ status_url = conf.OPA_STATUS_API + '/osdu/partition/' + auth_data.data_partition_id
r_status = requests.get(status_url, timeout=10, headers=headers)
r_status_json = r_status.json()
except requests.exceptions.HTTPError:
diff --git a/app/api/policy_eval_api.py b/app/api/policy_eval_api.py
index 81a69b969153ed342fde9221b4c9e2e918116169..47c6dea036fba215aab87646f15c9dd9bd2ad366 100644
--- a/app/api/policy_eval_api.py
+++ b/app/api/policy_eval_api.py
@@ -66,6 +66,10 @@ def evaluate_policy(
default=False,
description="Update posted data to include auth (token, xuserid and data partition id) from headers",
),
+ cache: bool = Query(
+ default=True,
+ description="Use cache (if enabled) on policy service.",
+ ),
):
"""
## Evaulate Policies
@@ -105,6 +109,8 @@ def evaluate_policy(
### Permission required to use this API:
`policy.service.user` or `policy.service.admin`
+
+ Setting cache to disable is intended for testing policies, in particular policies that do caching in OPA.
"""
logging.setLogRecordFactory(
correlation.set_correlation_id(context["correlation_id"])
@@ -125,7 +131,7 @@ def evaluate_policy(
posted_data = process_posted_data(
file=file, include_auth=include_auth, auth_data=auth_data
)
- result = opa.data(query=json.dumps(posted_data), path=policy_id)
+ result = opa.data(query=json.dumps(posted_data), path=policy_id, cache=cache)
if result.ok:
logging.debug(result.message)
diff --git a/app/opa.py b/app/opa.py
index 3f97419e3d6419bc85eadcdf1eff434b9eff79bb..a19de2360adaf91774aa0e4c290e407952294de7 100644
--- a/app/opa.py
+++ b/app/opa.py
@@ -278,15 +278,17 @@ def compile_with_caching(query, metrics=False, instrument=False, timeout=20):
return result
-def data(query, path, timeout=20, name="opa data api"):
+def data(query, path, timeout=20, name="opa data api", cache=True):
logging.setLogRecordFactory(
correlation.set_correlation_id(context["correlation_id"])
)
logger = logging.getLogger(__name__)
t = timer.Timer(name, logger=logger.debug)
t.start()
- result = data_with_caching(query=query, path=path, timeout=timeout)
- logger.debug(f"data opa cache info: {data_with_caching.cache_info()}")
+ if cache:
+ result = data_with_caching(query=query, path=path, timeout=timeout)
+ else:
+ result = internal_data(query=query, path=path, timeout=timeout)
t.stop()
return result
@@ -297,6 +299,9 @@ def data(query, path, timeout=20, name="opa data api"):
), info=conf.OPA_DATA_CACHE_INFO
)
def data_with_caching(query, path, timeout=20):
+ return internal_data(query, path, timeout)
+
+def internal_data(query, path, timeout=20):
"""
Compile - Partially Evaluate a Query.
The Compile API allows you to partially evaluate Rego queries and obtain a simplified version of the policy.
diff --git a/devops/aws/opa/templates/configmap.yaml b/devops/aws/opa/templates/configmap.yaml
index 36e9e372f4fd50e5ceca6f888897c272efd6c0ea..de0ee7708f1127fdaa1ebe2b8a5c7ffaf1f6c069 100644
--- a/devops/aws/opa/templates/configmap.yaml
+++ b/devops/aws/opa/templates/configmap.yaml
@@ -21,6 +21,7 @@ metadata:
name: opa-config
data:
config: |
+ caching.inter_query_builtin_cache.max_size_bytes: 943718400
services:
s3:
url: {{ .Values.policyBucket }}
diff --git a/docs/docs/api.md b/docs/docs/api.md
index ea5c20ae10de01719e77a8e72bbc717a8ea76402..051f91ec6a9f4d78784cea844ad6e0f15a75a3b9 100644
--- a/docs/docs/api.md
+++ b/docs/docs/api.md
@@ -84,6 +84,8 @@ curl -X 'DELETE' \
### POST /api/policy/v1/evaluations/query
Evaluates the provided policy (referenced with _policy_id_) with the provided input. The payload for the evaluation contains a file with the _policy_id_ and _input_ for evaluation. Input contains the _operation_, _record_ or list of _records_, _groups_ that the user is a member of, _user_ attributes that can be used in a policy definition, and _legaltags_ that contain legal attributes for record(s).
+
+Evaluations/query also allows you to by-pass policy service cache. This can be useful if you are testing policies that do OPA based caching.
<details>
For example file data for policy dataauthz.rego: Where XXXX is the data partition and YYYY is a legal tag
diff --git a/docs/docs/cache.md b/docs/docs/cache.md
index 231db620989d9ce4ac41033e16b9689e2b5eee59..562d187ea9f55ffc49034dc87a969df2466dbabd 100644
--- a/docs/docs/cache.md
+++ b/docs/docs/cache.md
@@ -1,19 +1,19 @@
# Cache
-## OPA Caching
+## OPA Caching from Policy
-Caching of [OPA](https://www.openpolicyagent.org/) responses was added in M16 to handle cases where OPA was undersized in terms of memory requests and number of replicas (pods). Caching should not be needed if OPA is sized large enough for your load.
+Caching of [OPA](https://www.openpolicyagent.org/) responses was added in M16 to handle cases where OPA was undersized in terms of memory requests and number of replicas (pods). Caching should not be needed if OPA is sized large enough for your load, however you'll likely get better performance with caching enabled.
When search makes calls to Policy Service Translate API, Policy Service can make 1-2 calls to OPA per search request. There are also other APIs in policy that make calls to OPA and there are services that talk to OPA directly like search.
See [sizing](sizing.md) for more details on OPA Sizing recommendations.
-### How to disable
+### How to disable Policy Service Cache
To disable caching set environment variable `DISABLE_OPA_CACHE` to `True`.
Caching will also be disabled if the `OPA_CACHE_MAXSIZE` is 0
-## Adjusting Cache Settings
+## Adjusting Policy Service Cache Settings
Caching of OPA responses can be controlled by the following environments variables.
Shown with defaults:
```
@@ -73,7 +73,15 @@ To use caching with rego policies in [http.send](https://www.openpolicyagent.org
If you use `force_cache`, please note this overrides cache directives defined by the OPA service. So you'll want to set `force_cache_duration_seconds`.
-!!! warning
+!!! warning "cache_ignored_headers"
+
If a cache entry exists with a subset/superset of headers that are considered in this request, it will lead to a cache miss. Please consider using `cache_ignored_headers` when using caching. Please note, `cache_ignored_headers` was added in v0.66 release of OPA.
-You'll also want to consider using `caching.inter_query_builtin_cache.max_size_bytes` to set a Inter-query cache size limit in bytes. OPA will drop old items from the cache if this limit is exceeded. By default, no limit is set (i.e. unlimited) and this could cause Out of Memory `OOMKilled` and `MemoryPressure` issues. This is an OPA config setting (not in Rego).
\ No newline at end of file
+## OPA Cache Settings
+
+!!! warning "Set OPA default cache settings"
+
+ If you are using cache settings in policies it is strongly recommended to consider using these OPA config settings(not in Rego).
+- `caching.inter_query_builtin_cache.max_size_bytes` to set a Inter-query cache size limit in bytes. OPA will drop old items from the cache if this limit is exceeded. By default, no limit is set (i.e. unlimited) and this could cause Out of Memory `OOMKilled` and `MemoryPressure` issues.
+- `caching.inter_query_builtin_cache.forced_eviction_threshold_percentage` - Threshold limit configured as percentage of caching.inter_query_builtin_cache.max_size_bytes, when exceeded OPA will start dropping old items permaturely. By default, set to 100.
+- `caching.inter_query_builtin_cache.stale_entry_eviction_period_seconds` - Stale entry eviction period in seconds. OPA will drop expired items from the cache every stale_entry_eviction_period_seconds. By default, set to 0 indicating stale entry eviction is disabled.
\ No newline at end of file
diff --git a/docs/docs/testing.md b/docs/docs/testing.md
index b174a49b7c7f755c9a7ca7ec45027401d9844b93..d058cbbfb7c68d722bbbda1ddd0abc1e33992867 100644
--- a/docs/docs/testing.md
+++ b/docs/docs/testing.md
@@ -47,6 +47,11 @@ The [test directory](https://community.opengroup.org/osdu/platform/security-and-
* `OPA_COMPILE_CACHE_TTL` - Added in M20
* `OPA_DATAAPI_CACHE_MAXSIZE` - Added in M20
* `OPA_DATAAPI_CACHE_TTL` - Added in M20
+* `OPA_DATA_CACHE_INFO` - Added in M25
+* `OPA_COMPILE_CACHE_INFO` - Added in M25
+* `OPA_DOCUMENT_CACHE_INFO` - Added in M25
+* `OPA_FETCH_CACHE_INFO` - Added in M25
+* `OPA_LIST_CACHE_INFO` - Added in M25
### BUNDLE_PAUSE