diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2a2e94f8a4098b647b7e5e659a641b3ff1b6cd92..0800a35805d41855a61140b9939f0acd8079f4fc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -92,6 +92,11 @@ build-image:
- echo ---- ---- ---- SYSTEM DEPENDENCIES ---- ---- ----
- apk update
- apk add git
+ - echo ---- ---- ---- PYTHON DEPENDENCIES ---- ---- ----
+ - apk add --no-cache python3 py3-pip
+ - pip install -r requirements_setversion.txt
+ - echo ---- ---- ---- UPDATE BUILD INFO---- ---- ----
+ - version=$(python3 setversion.py app)
- echo ---- ---- ---- BUILD IMAGE ---- ---- ----
- echo IMAGE_TAG $IMAGE_TAG
- commit_id=$(git log -n 1 --pretty="%h")
@@ -106,12 +111,14 @@ build-image:
- echo commit_branch $commit_branch
- COMMIT_IMAGE_NAMETAG=${CI_REGISTRY_IMAGE}:${commit_id}
- BRANCH_IMAGE_NAMETAG=${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}
+ - VERSION_IMAGE_NAMETAG=${CI_REGISTRY_IMAGE}:${version}
- echo ---- ---- BUILD IMAGE
- - docker build -t $IMAGE_TAG -t=$CI_REGISTRY_IMAGE:latest -t $COMMIT_IMAGE_NAMETAG -t $BRANCH_IMAGE_NAMETAG --rm . -f ./build/Dockerfile --build-arg PIP_WHEEL_DIR=python-packages --build-arg build_date="$current_utc_date" --build-arg build_number=$commit_id --build-arg commit_id=$commit_id --build-arg build_origin="Gitlab" --build-arg commit_branch=$commit_branch
+ - docker build -t $IMAGE_TAG -t=$CI_REGISTRY_IMAGE:latest -t $COMMIT_IMAGE_NAMETAG -t $BRANCH_IMAGE_NAMETAG -t $VERSION_IMAGE_NAMETAG --rm . -f ./build/Dockerfile --build-arg PIP_WHEEL_DIR=python-packages --build-arg build_date="$current_utc_date" --build-arg build_number=$commit_id --build-arg commit_id=$commit_id --build-arg build_origin="Gitlab" --build-arg commit_branch=$commit_branch
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- echo ---- ---- PUSH IMAGE $IMAGE_TAG
- docker push $IMAGE_TAG
- echo ---- ---- PUSH IMAGE $CI_REGISTRY_IMAGE:latest
- docker push $CI_REGISTRY_IMAGE:latest
+ - docker push $VERSION_IMAGE_NAMETAG
- docker push $COMMIT_IMAGE_NAMETAG
- docker push $BRANCH_IMAGE_NAMETAG
diff --git a/NOTICE b/NOTICE
index 86a668142f2b22e971cb5a2aa17bef78920ff1d3..03178a9d1bf0dc023c5e5e380f30cfcc4ea964d6 100644
--- a/NOTICE
+++ b/NOTICE
@@ -41,7 +41,7 @@ The following software have components provided under the terms of this license:
- sniffio (from https://github.com/python-trio/sniffio)
- tenacity (from https://github.com/jd/tenacity)
- types-PyYAML (from https://github.com/python/typeshed)
-- urllib3 (from https://pypi.org/project/urllib3/2.0.2/, https://pypi.org/project/urllib3/2.0.4/, https://pypi.org/project/urllib3/2.0.6/)
+- urllib3 (from https://pypi.org/project/urllib3/2.0.2/, https://pypi.org/project/urllib3/2.0.4/, https://pypi.org/project/urllib3/2.0.7/)
- websocket-client (from https://github.com/websocket-client/websocket-client.git)
========================================================================
@@ -96,7 +96,7 @@ The following software have components provided under the terms of this license:
- PyJWT (from http://github.com/jpadilla/pyjwt, https://github.com/jpadilla/pyjwt)
- PyYAML (from http://pyyaml.org/wiki/PyYAML)
-- annotated-types (from https://pypi.org/project/annotated-types/0.5.0/)
+- annotated-types (from https://pypi.org/project/annotated-types/0.5.0/, https://pypi.org/project/annotated-types/0.6.0/)
- anyio (from https://pypi.org/project/anyio/3.3.0/, https://pypi.org/project/anyio/4.0.0/)
- attrs (from https://attrs.readthedocs.io/, https://pypi.org/project/attrs/23.1.0/, https://www.attrs.org/)
- azure-common (from https://github.com/Azure/azure-sdk-for-python)
@@ -110,7 +110,7 @@ The following software have components provided under the terms of this license:
- charset-normalizer (from https://github.com/Ousret/charset_normalizer)
- coverage (from https://github.com/nedbat/coveragepy)
- exceptiongroup (from https://pypi.org/project/exceptiongroup/1.0.1/, https://pypi.org/project/exceptiongroup/1.1.3/)
-- fastapi (from https://pypi.org/project/fastapi/0.103.2/, https://pypi.org/project/fastapi/0.86.0/)
+- fastapi (from https://pypi.org/project/fastapi/0.104.0/, https://pypi.org/project/fastapi/0.86.0/)
- h11
- iniconfig (from http://github.com/RonnyPfannschmidt/iniconfig, https://pypi.org/project/iniconfig/2.0.0/)
- jmespath (from https://github.com/jmespath/jmespath.py)
@@ -133,7 +133,8 @@ The following software have components provided under the terms of this license:
- sniffio (from https://github.com/python-trio/sniffio)
- starlette-context (from https://github.com/tomwojcik/starlette-context)
- tomli (from https://pypi.org/project/tomli/1.2.2/, https://pypi.org/project/tomli/2.0.0/, https://pypi.org/project/tomli/2.0.1/)
-- urllib3 (from https://pypi.org/project/urllib3/2.0.2/, https://pypi.org/project/urllib3/2.0.4/, https://pypi.org/project/urllib3/2.0.6/)
+- typer (from https://github.com/tiangolo/typer)
+- urllib3 (from https://pypi.org/project/urllib3/2.0.2/, https://pypi.org/project/urllib3/2.0.4/, https://pypi.org/project/urllib3/2.0.7/)
- uuid7 (from https://github.com/stevesimmons/uuid7)
========================================================================
diff --git a/VERSION b/VERSION
new file mode 100644
index 0000000000000000000000000000000000000000..d21d277be513781949acc78323bca782b467679d
--- /dev/null
+++ b/VERSION
@@ -0,0 +1 @@
+0.25.0
diff --git a/app/Makefile b/app/Makefile
index 67805fcbfa628cf229e91ef8f777117f9c2a714b..6e6a3ef9d20c3618d6f8500068a8527b83c4f244 100644
--- a/app/Makefile
+++ b/app/Makefile
@@ -1,4 +1,4 @@
-+IMAGE_NAME=policy-service
+IMAGE_NAME=policy-service
BASE_IMAGE_NAME=python:3.9-slim-buster
CONTAINER_NAME=policy-service-source
PYTHON=/opt/homebrew/bin/python3.9
@@ -18,7 +18,7 @@ OPA_PORT=8181
.EXPORT_ALL_VARIABLES:
POLICY_BUCKET=r3m16-515517605230-us-east-1-policy
GOOGLE_CLOUD_PROJECT=osdu-dev-policy
-DATA_PARTITION=osdu
+DATA_PARTITION := ${DATA_PARTITION}
OPA_URL = http://$(OPA_NAME):$(OPA_PORT)
mkfile_path := $(abspath $(lastword $(MAKEFILE_LIST)))
current_dir := $(notdir $(patsubst %/,%,$(dir $(mkfile_path))))
@@ -39,6 +39,7 @@ aws_set_token_green:
build: build_adminui build_docker
build_docker:
+ cd .. && python3 setversion.py app
cd .. && docker build --network host -t $(IMAGE_NAME):$(TAG) -f app/Dockerfile .
build_adminui:
@@ -124,7 +125,7 @@ local: ENABLE_VERIFY_POLICY := True
local: echoenv
local:
#- uvicorn main:app --port $(PORT) --host 0.0.0.0 --workers $(WORKERS) --lifespan on --loop uvloop
- - uvicorn main:app --port $(PORT) --host 0.0.0.0 --lifespan on --loop uvloop --reload --proxy-headers
+ - $(PYTHON) -m uvicorn main:app --port $(PORT) --host 0.0.0.0 --lifespan on --loop uvloop --reload --proxy-headers
loadtest: LOG_LEVEL := ERROR
loadtest: OPA_URL := http://localhost:$(OPA_PORT)
diff --git a/app/_buildinfo.py b/app/_buildinfo.py
deleted file mode 100644
index 9f5f1c2184964021121658e70b8c8512581e5ede..0000000000000000000000000000000000000000
--- a/app/_buildinfo.py
+++ /dev/null
@@ -1,11 +0,0 @@
-# IN THE FUTURE THIS WILL BE GENERATED BY CI
-version = "0.24.0"
-milestone = "M21"
-artifactId = None
-name = "policy"
-groupId = "org.opengroup.osdu"
-buildTime = None
-branch = "master"
-commitId = None
-commitMessage = "M21"
-buildTime = None
\ No newline at end of file
diff --git a/app/_version.py b/app/_version.py
new file mode 100644
index 0000000000000000000000000000000000000000..6e2c568cb1414388eba977bf4892652e02c62c0b
--- /dev/null
+++ b/app/_version.py
@@ -0,0 +1,8 @@
+__version__ = "0.24.a0.dev0"
+__milestone__ = "M21"
+__branch__ = "dev"
+__buildtime__ = 1698466844.468756
+__commitid__ = ""
+__commitmessage__ = ""
+__committimestamp__ = "2023-10-28T00:20:44.468643"
+__commitrefslug__ = ""
diff --git a/app/api/backup_api.py b/app/api/backup_api.py
index ea95e75ceec4b0b964516b881c48b17e8b4b044e..763356f189ae9a0be1ab8a33f8d8002ee55915cc 100644
--- a/app/api/backup_api.py
+++ b/app/api/backup_api.py
@@ -1,24 +1,30 @@
import fastapi
from fastapi import Depends, HTTPException, Response
-import requests
from auth import auth
import logging
import conf
from starlette_context import context
from fastapi.responses import StreamingResponse
from bundles import bundle
-router = fastapi.APIRouter()
import correlation
from datetime import date
+from pydantic import BaseModel
from starlette.status import (
- HTTP_405_METHOD_NOT_ALLOWED,
+ HTTP_400_BAD_REQUEST,
HTTP_424_FAILED_DEPENDENCY
)
+router = fastapi.APIRouter()
+
+class Detail(BaseModel):
+ detail: str
_logger = logging.getLogger(__name__)
-@router.get("/backup")
+@router.get("/backup", responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_424_FAILED_DEPENDENCY: {"model": Detail},
+ })
def backup(response: Response, auth_data: auth.Auth = Depends(auth.require_authorized_admin)):
"""
Experimental Backup API.
diff --git a/app/api/bootstrap_api.py b/app/api/bootstrap_api.py
index 5ebdaf8fc2a90f877d62455801f47df951b773ce..d04baa62d4cbc4087d146bbebe3b61df91e06999 100644
--- a/app/api/bootstrap_api.py
+++ b/app/api/bootstrap_api.py
@@ -6,19 +6,30 @@ import conf
import json
from starlette_context import context
from bundles import bundle
-router = fastapi.APIRouter()
import correlation
+from pydantic import BaseModel
from starlette.status import (
HTTP_201_CREATED,
HTTP_202_ACCEPTED,
+ HTTP_400_BAD_REQUEST,
HTTP_405_METHOD_NOT_ALLOWED,
HTTP_424_FAILED_DEPENDENCY
)
+class Detail(BaseModel):
+ detail: str
+
+router = fastapi.APIRouter()
+
logger = logging.getLogger(__name__)
-@router.post("/bootstrap")
+@router.post("/bootstrap", responses={
+ HTTP_201_CREATED: {"model": Detail},
+ HTTP_202_ACCEPTED: {"model": Detail},
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_405_METHOD_NOT_ALLOWED: {"model": Detail}
+ })
def bootstrap(response: Response,
auth_data: auth.Auth = Depends(auth.require_authorized_admin),
force: bool = False
@@ -46,15 +57,16 @@ def bootstrap(response: Response,
response.headers["X-Correlation-ID"] = context["correlation_id"]
if bundle.get_bundle(data_partition=auth_data.data_partition_id):
- logger.debug(f"bootstrap bundle already exists")
+ logger.debug("bootstrap bundle already exists")
if force:
if not conf.ALLOW_FORCE_BOOTSTRAP:
raise HTTPException(status_code=HTTP_405_METHOD_NOT_ALLOWED, detail=f"'{auth_data.data_partition_id}' already bootstrapped and forced updates not allowed.")
status = bootstrap_bundle(data_partition=auth_data.data_partition_id)
if status:
response.status_code = HTTP_202_ACCEPTED
- detail = f"AUDIT bootstrap bundle for partition '{auth_data.data_partition_id}' updated to default, user: {auth_data.user_id}"
- logger.info(detail)
+ detail = f"bootstrap bundle for partition '{auth_data.data_partition_id}' updated to default"
+ audit = f"AUDIT {detail}, user: {auth_data.user_id}"
+ logger.info(audit)
return {"detail": detail}
else:
raise HTTPException(status_code=HTTP_424_FAILED_DEPENDENCY, detail=f"'{auth_data.data_partition_id}' update failed {status}!")
@@ -64,8 +76,9 @@ def bootstrap(response: Response,
status = bootstrap_bundle(data_partition=auth_data.data_partition_id)
if status:
response.status_code = HTTP_201_CREATED
- detail = f"AUDIT bootstrap bundle for partition '{auth_data.data_partition_id}' created, user: {auth_data.user_id}."
- logger.info(detail)
+ detail = f"bootstrap bundle for partition '{auth_data.data_partition_id}' created"
+ audit = f"AUDIT {detail}, user: {auth_data.user_id}"
+ logger.info(audit)
return {"detail": detail}
else:
detail = f"bootstrap bundle for partition '{auth_data.data_partition_id}' create failed {status}!"
diff --git a/app/api/compile_api.py b/app/api/compile_api.py
index 2bfff8ca3cf2284a2eea97c2b220c18f5ef995b3..b5d95dab13562b38d1291bfb77cb44871f032c9c 100644
--- a/app/api/compile_api.py
+++ b/app/api/compile_api.py
@@ -1,23 +1,26 @@
-from fastapi import APIRouter, Depends, File, UploadFile, HTTPException, Query, Response
+from fastapi import APIRouter, Depends, UploadFile, HTTPException, Query, Response
from starlette_context import context
-import os
import logging
-from requests.structures import CaseInsensitiveDict
from auth import auth
-import json
-import requests
-router = APIRouter()
-import conf
-import _buildinfo as b
import opa
import correlation
+from pydantic import BaseModel
from starlette.status import (
+ HTTP_400_BAD_REQUEST,
HTTP_422_UNPROCESSABLE_ENTITY,
)
+
+class Detail(BaseModel):
+ detail: str
+
+router = APIRouter()
logger = logging.getLogger(__name__)
-@router.post("/compile")
+@router.post("/compile", responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_422_UNPROCESSABLE_ENTITY: {"model": Detail},
+ })
def compile_partially_evaluate_a_query(
response: Response,
file: UploadFile,
diff --git a/app/api/config_api.py b/app/api/config_api.py
index 54c04ae48de0e948661734fe9056d265bcb14559..66024e494c164e068d76acd41ea570b17e5b9894 100644
--- a/app/api/config_api.py
+++ b/app/api/config_api.py
@@ -1,16 +1,13 @@
-from typing import Annotated, List, Union
-from fastapi import APIRouter, Depends, Response, Request, Header
+from fastapi import APIRouter, Depends, Response, Request
from starlette_context import context
import os
import logging
from requests.structures import CaseInsensitiveDict
from auth import auth
-import json
import requests
-router = APIRouter()
import conf
-import _buildinfo as b
-import opa
+#import _buildinfo as b
+import _version as b
import correlation
import socket
import k8s
@@ -18,14 +15,26 @@ import time
import psutil
#import pdb
+from pydantic import BaseModel
+
+from starlette.status import (
+ HTTP_400_BAD_REQUEST,
+)
+router = APIRouter()
+
+class Detail(BaseModel):
+ detail: str
+
def seconds_elapsed():
return time.time() - psutil.boot_time()
-@router.get('/config')
+@router.get('/config', responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ })
def show_policy_config_details(
response: Response,
request: Request,
- duplicate: Annotated[Union[List[str], None], Header()] = None, # Python 3.9+
+ #duplicate: Annotated[Union[List[str], None], Header()] = None, # Python 3.9+
auth_data: auth.Auth = Depends(auth.require_authorized_admin)):
"""
Return detail configuration details.
@@ -54,7 +63,7 @@ def show_policy_config_details(
r_status_json = r_status.json()
except requests.exceptions.HTTPError:
logger.error(f"Error: endpoint {status_url}: HTTPError")
- except:
+ except Exception:
logger.error(f"Error: endpoint {status_url}: Error")
try:
@@ -63,7 +72,7 @@ def show_policy_config_details(
r_config_headers = r_config.headers
except requests.exceptions.HTTPError:
logger.error(f"Error: endpoint {conf.OPA_CONFIG_API}: HTTPError")
- except:
+ except Exception:
logger.error(f"Error: endpoint {conf.OPA_CONFIG_API}: Error")
cpu_usage = None
@@ -87,7 +96,7 @@ def show_policy_config_details(
conf_dict = {}
for i in dir(conf):
if i[0].isupper():
- if not "MOCK_ENTITLEMENT_RESULT" in i and not "TOKEN" in i:
+ if "MOCK_ENTITLEMENT_RESULT" not in i and "TOKEN" not in i:
conf_dict[i] = getattr(conf, i)
if conf.MASK_OPA_CONFIG_DETAILS:
@@ -125,21 +134,22 @@ def show_policy_config_details(
"memory_usage": memory_usage,
"client_host": request.client.host,
"request_headers": request.headers,
- "duplicate": duplicate,
+ #"duplicate": duplicate,
"auth_data": {
"data_partition_id": auth_data.data_partition_id,
# "groups": auth_data.groups
},
"build_data": {
- "version": b.version,
- "milestone": b.milestone,
- "artifactId": b.artifactId,
- "name": b.name,
- "groupId": b.groupId,
- "buildTime": b.buildTime,
- "branch": b.branch,
- "commitId": b.commitId,
- "commitMessage": b.commitMessage
+ "version": b.__version__,
+ "milestone": b.__milestone__,
+ "artifactId": None,
+ "name": "policy",
+ "groupId": "org.opengroup.osdu",
+ "buildTime": b.__committimestamp__,
+ "branch": b.__branch__,
+ "commitId": b.__commitid__,
+ "commitMessage": b.__commitmessage__,
+ "commitrefslug": b.__commitrefslug__
},
"opa_status": r_status_json,
"opa_config": r_config_json
diff --git a/app/api/diag_api.py b/app/api/diag_api.py
index 3b5d753f1b29fcca8b60d31ee48682062fc0afaa..4f810430f6e1cfcfea3d0cdf7e848c5ba7ffaa31 100644
--- a/app/api/diag_api.py
+++ b/app/api/diag_api.py
@@ -1,17 +1,11 @@
-from fastapi import APIRouter, Depends, File, UploadFile, HTTPException, Query
+from fastapi import APIRouter, Depends, HTTPException
from starlette_context import context
-import os
import logging
-from requests.structures import CaseInsensitiveDict
from auth import auth
-import json
-import requests
-router = APIRouter()
import conf
-import _buildinfo as b
import opa
import correlation
-import inspect
+router = APIRouter()
logger = logging.getLogger(__name__)
if conf.ENABLE_DEV_DIAGNOSTICS:
diff --git a/app/api/health_api.py b/app/api/health_api.py
index 1cd75a0e1d72c6521b80fc6036321c098a006fe5..355d52111bd828ff2189e91561ce8c45c99647f5 100644
--- a/app/api/health_api.py
+++ b/app/api/health_api.py
@@ -1,16 +1,24 @@
from fastapi import APIRouter, HTTPException
import requests
import logging
+from pydantic import BaseModel
from starlette.status import (
+ HTTP_400_BAD_REQUEST,
HTTP_501_NOT_IMPLEMENTED,
HTTP_503_SERVICE_UNAVAILABLE
)
import conf
+class Detail(BaseModel):
+ detail: str
+
router = APIRouter()
-@router.get("/health")
+@router.get("/health", responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_503_SERVICE_UNAVAILABLE: {"model": Detail}
+ })
async def health():
"""
## Health check endpoint, which does not depend on OPA.
@@ -22,7 +30,11 @@ async def health():
"""
return {'message': 'Healthy'}
-@router.get("/ready")
+@router.get("/ready", responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_501_NOT_IMPLEMENTED: {"model": Detail},
+ HTTP_503_SERVICE_UNAVAILABLE: {"model": Detail},
+ })
def ready():
"""
## Health check endpoint, which depends on OPA being available and healthy.
@@ -39,10 +51,10 @@ def ready():
try:
url = conf.OPA_HEALTH_API
except NameError:
- logging.error(f"conf.OPA_URL undefined")
+ logging.error("conf.OPA_URL undefined")
raise HTTPException(status_code=HTTP_501_NOT_IMPLEMENTED, detail="NotHealthy - OPA configuration issue")
except AttributeError:
- logging.error(f"conf.OPA_URL undefined")
+ logging.error("conf.OPA_URL undefined")
raise HTTPException(status_code=HTTP_501_NOT_IMPLEMENTED, detail="NotHealthy - OPA configuration issue")
try:
diff --git a/app/api/info_api.py b/app/api/info_api.py
index 03c5cf92a8f54f83ebfcec78870215ac5e42a76e..e21bd5667868c0727cd32b78d1f74a9aa96a8a6e 100644
--- a/app/api/info_api.py
+++ b/app/api/info_api.py
@@ -4,14 +4,28 @@ import requests
import logging
import conf
from models.info import InfoOut, ServiceDetail, Services
-import _buildinfo as b
+import _version as b
from starlette_context import context
-router = fastapi.APIRouter()
+from pydantic import BaseModel
import correlation
+from starlette.status import (
+ HTTP_400_BAD_REQUEST,
+ HTTP_503_SERVICE_UNAVAILABLE
+)
+
+router = fastapi.APIRouter()
+
+class Detail(BaseModel):
+ detail: str
+
logger = logging.getLogger(__name__)
-@router.get("/info", response_model=InfoOut)
+@router.get("/info", response_model=InfoOut,
+ responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_503_SERVICE_UNAVAILABLE: {"model": Detail}
+ })
def return_version_info(response: Response):
"""
Return Service version information.
@@ -43,14 +57,14 @@ def return_version_info(response: Response):
svcdetail = ServiceDetail(version=opa_version)
opa = Services(opa=svcdetail)
info = InfoOut(
- version = b.version,
- artifactId = b.artifactId,
- name = b.name,
- groupId = b.groupId,
- buildTime = b.buildTime,
- branch = b.branch,
- commitId = b.commitId,
- commitMessage = b.commitMessage,
+ version = b.__version__,
+ artifactId = None,
+ name = "policy",
+ groupId = "org.opengroup.osdu",
+ buildTime = b.__committimestamp__,
+ branch = b.__branch__,
+ commitId = b.__commitid__,
+ commitMessage = b.__commitmessage__,
connectedOuterServices=opa
)
return info
\ No newline at end of file
diff --git a/app/api/policy_eval_api.py b/app/api/policy_eval_api.py
index 3b06f6a40747637061af4e6f4cc0101f3f5d2aa1..d428d83f1bb536d41ee090858e806d3d59ce2c98 100644
--- a/app/api/policy_eval_api.py
+++ b/app/api/policy_eval_api.py
@@ -14,7 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-from email import header
import fastapi
from fastapi import Depends, HTTPException, UploadFile, Query, Response
import sys
@@ -22,28 +21,37 @@ import os
import logging
import json
from starlette_context import context
+from pydantic import BaseModel
import conf
import correlation
from starlette.status import (
- HTTP_200_OK,
- HTTP_403_FORBIDDEN,
+ HTTP_400_BAD_REQUEST,
HTTP_406_NOT_ACCEPTABLE,
HTTP_422_UNPROCESSABLE_ENTITY,
+ HTTP_500_INTERNAL_SERVER_ERROR,
+ HTTP_503_SERVICE_UNAVAILABLE
)
-from opa_response import OpaResponse
+#from opa_response import OpaResponse
sys.path.append(os.path.abspath('..'))
import opa
from auth import auth
+class Detail(BaseModel):
+ detail: str
+
logger = logging.getLogger(__name__)
router = fastapi.APIRouter()
# Header code is compatible with Python 3.6 - 3.9, it will not work with Python 3.10
# See https://fastapi.tiangolo.com/tutorial/header-params/
-@router.post("/evaluations/query")
+@router.post("/evaluations/query", responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_500_INTERNAL_SERVER_ERROR: {"model": Detail},
+ HTTP_503_SERVICE_UNAVAILABLE: {"model": Detail},
+ })
def evaluate_policy(
response: Response,
policy_id: str,
@@ -96,7 +104,7 @@ def evaluate_policy(
# Try to make it easier for developers
if 'osdu/instance/' in policy_id:
logger.debug(f"Instance policy eval {policy_id}")
- elif not 'osdu/partition/' in policy_id:
+ elif 'osdu/partition/' not in policy_id:
policy_id = 'osdu/partition/' + auth_data.data_partition_id + '/' + policy_id
#contents = await file.read()
@@ -104,8 +112,8 @@ def evaluate_policy(
try:
posted_data = json.loads(contents.decode('utf-8'))
- if not 'input' in posted_data:
- raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Input not found in file data")
+ if 'input' not in posted_data:
+ raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail="Input not found in file data")
except json.decoder.JSONDecodeError as err:
raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=f"JSON error {err}")
except UnicodeDecodeError as err:
@@ -120,10 +128,10 @@ def evaluate_policy(
posted_data["input"]["datapartitionid"] = auth_data.data_partition_id
# make sure we have them
- if not 'token' in posted_data["input"]:
- raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=f"token not found in file data")
- if not 'datapartitionid' in posted_data["input"]:
- raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=f"datapartitionid not found in file data")
+ if 'token' not in posted_data["input"]:
+ raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail="token not found in file data")
+ if 'datapartitionid' not in posted_data["input"]:
+ raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail="datapartitionid not found in file data")
result = opa.data(query=json.dumps(posted_data), path=policy_id)
logging.error(result)
@@ -133,7 +141,7 @@ def evaluate_policy(
logging.info(result.message)
raise HTTPException(status_code=result.status_code, detail=f"Error when talking to OPA: {result.message}")
- if not "result" in result.json:
+ if "result" not in result.json:
if conf.ENABLE_DEV_DIAGNOSTICS_DUMPS:
# save lots of debugging info
file_path = "eval_dump_bad.json"
diff --git a/app/api/policy_read_api.py b/app/api/policy_read_api.py
index 97d53a02a225280619ff4e54f21f65562b6e2f3d..690f7594e392a06937582b58cef2f6dea71d8d4c 100644
--- a/app/api/policy_read_api.py
+++ b/app/api/policy_read_api.py
@@ -21,6 +21,8 @@ import sys
import os
import logging
from starlette_context import context
+import hashlib
+from pydantic import BaseModel
from starlette.status import (
HTTP_400_BAD_REQUEST,
@@ -32,10 +34,11 @@ from opa_response import OpaResponse
sys.path.append(os.path.abspath('..'))
import conf
import opa
-from datetime import date
from auth import auth
import correlation
-import hashlib
+
+class Detail(BaseModel):
+ detail: str
logger = logging.getLogger(__name__)
router = fastapi.APIRouter()
@@ -43,7 +46,11 @@ router = fastapi.APIRouter()
# Header code is compatible with Python 3.6 - 3.9, it will not work with Python 3.10
# See https://fastapi.tiangolo.com/tutorial/header-params/
-@router.get("/policies")
+@router.get("/policies",
+ responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_500_INTERNAL_SERVER_ERROR: {"model": Detail},
+ })
def fetch_all_policies_that_match_partition(response: Response, data_partition_id: Union[str, None] = Header(default=None), auth_data: auth.Auth = Depends(auth.require_authorized_user)):
"""
Return all policies from OPA directly that match partition_bundle_root data-partition-id in header (if bundles are enabled).
@@ -58,11 +65,11 @@ def fetch_all_policies_that_match_partition(response: Response, data_partition_i
cloud_provider = os.environ.get('CLOUD_PROVIDER')
if cloud_provider is None:
opa_response = OpaResponse()
- logger.critical(f"Error: CLOUD_PROVIDER ENV VAR not set / Mocking results for /policies")
+ logger.critical("Error: CLOUD_PROVIDER ENV VAR not set / Mocking results for /policies")
opa_response.json = {"result": {}}
elif cloud_provider == conf.MOCK:
opa_response = OpaResponse()
- logger.critical(f"Warning: CLOUD_PROVIDER ENV VAR set to Mock results for /policies")
+ logger.critical("Warning: CLOUD_PROVIDER ENV VAR set to Mock results for /policies")
opa_response.json = {"result": {}}
else:
opa_response = opa.fetch_opa_policies_direct()
@@ -80,7 +87,12 @@ def fetch_all_policies_that_match_partition(response: Response, data_partition_i
x['id'].startswith('/' + conf.INSTANCE_BUNDLE_ROOT) ]}
raise HTTPException(status_code=HTTP_500_INTERNAL_SERVER_ERROR, detail="Unexpected result from OPA")
-@router.get("/policies/{policy_id}")
+@router.get("/policies/{policy_id}",
+ responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_404_NOT_FOUND: {"model": Detail},
+ HTTP_500_INTERNAL_SERVER_ERROR: {"model": Detail},
+ })
def fetch_a_policy(response: Response, policy_id: str, auth_data: auth.Auth = Depends(auth.require_authorized_user)):
"""
Return a policy directly from OPA with no filtering
@@ -108,11 +120,15 @@ def fetch_a_policy(response: Response, policy_id: str, auth_data: auth.Auth = D
else:
opa_response = opa.fetch_opa_policy_direct(policy_id)
- if opa_response.ok == True:
+ if opa_response.ok is True:
return opa_response.json
raise HTTPException(status_code=opa_response.status_code, detail=f"Unexpected result from OPA {opa_response.message}")
-@router.get("/policies/osdu/instance/{policy_id}")
+@router.get("/policies/osdu/instance/{policy_id}",
+ responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_500_INTERNAL_SERVER_ERROR: {"model": Detail},
+ })
def fetch_instance_policy(response: Response, policy_id: str, auth_data: auth.Auth = Depends(auth.require_authorized_user)):
"""
Return an instance policy from OPA directly.
@@ -141,11 +157,15 @@ def fetch_instance_policy(response: Response, policy_id: str, auth_data: auth.A
opa_response = opa.fetch_opa_policy_direct(conf.INSTANCE_BUNDLE_ROOT+"/{0}".format(policy_id))
logger.debug(f"OPA return {opa_response.json}")
- if opa_response.ok == True:
+ if opa_response.ok is True:
return opa_response.json
raise HTTPException(status_code=opa_response.status_code, detail=f"Unexpected result from OPA {opa_response.message}")
-@router.get("/policies/osdu/partition/{data_partition}/{policy_id}")
+@router.get("/policies/osdu/partition/{data_partition}/{policy_id}",
+ responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_500_INTERNAL_SERVER_ERROR: {"model": Detail},
+ })
def fetch_partition_policy_directly_from_opa(
response: Response,
policy_id: str,
@@ -191,7 +211,7 @@ def fetch_partition_policy_directly_from_opa(
# include extra debugging information in response headers
response.headers["X-Debug-Result"] = f"{opa_response.json}"
- if opa_response.ok == True:
+ if opa_response.ok is True:
if "result" in opa_response.json:
if "raw" in opa_response.json["result"]:
data = opa_response.json["result"]["raw"]
diff --git a/app/api/policy_update_api.py b/app/api/policy_update_api.py
index 49a4abd92b860382244cc08c6f3bd78e14cbff83..bbe1159e5d12c8e0f1ca5177a24fc887315b3b52 100644
--- a/app/api/policy_update_api.py
+++ b/app/api/policy_update_api.py
@@ -22,20 +22,24 @@ import logging
import hashlib
from starlette.requests import Request
from starlette_context import context
-from requests.structures import CaseInsensitiveDict
+#from requests.structures import CaseInsensitiveDict
from starlette.status import (
HTTP_200_OK,
+ HTTP_202_ACCEPTED,
HTTP_400_BAD_REQUEST,
+ HTTP_404_NOT_FOUND,
HTTP_422_UNPROCESSABLE_ENTITY,
- HTTP_501_NOT_IMPLEMENTED
+ HTTP_500_INTERNAL_SERVER_ERROR,
+ HTTP_501_NOT_IMPLEMENTED,
+ HTTP_503_SERVICE_UNAVAILABLE
)
+from pydantic import BaseModel
from opa_response import OpaResponse
sys.path.append(os.path.abspath('..'))
import conf
import opa
-from datetime import date
from auth import auth
from bundles import bundle
import correlation
@@ -44,10 +48,21 @@ from .validate_api import verify_policy_with_opa
logger = logging.getLogger(__name__)
router = fastapi.APIRouter()
+class Detail(BaseModel):
+ detail: str
+
# Header code is compatible with Python 3.6 - 3.9, it will not work with Python 3.10
# See https://fastapi.tiangolo.com/tutorial/header-params/
-@router.delete("/policies/osdu/partition/{data_partition}/{policy_id}")
+@router.delete("/policies/osdu/partition/{data_partition}/{policy_id}",
+ responses={
+ HTTP_202_ACCEPTED: {"model": Detail},
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_404_NOT_FOUND: {"model": Detail},
+ HTTP_501_NOT_IMPLEMENTED: {"model": Detail},
+ HTTP_503_SERVICE_UNAVAILABLE: {"model": Detail},
+ }
+ )
def delete_partition_policy(
response: Response,
request: Request,
@@ -156,7 +171,13 @@ def delete_partition_policy(
logger.error(f"Error when talking to bundle service {result.message}")
raise HTTPException(status_code=result.status_code, detail=f"Error when talking to bundle service {result.message}")
-@router.put("/policies/osdu/partition/{data_partition}/{policy_id}")
+@router.put("/policies/osdu/partition/{data_partition}/{policy_id}", responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_404_NOT_FOUND: {"model": Detail},
+ HTTP_500_INTERNAL_SERVER_ERROR: {"model": Detail},
+ HTTP_501_NOT_IMPLEMENTED: {"model": Detail},
+ HTTP_503_SERVICE_UNAVAILABLE: {"model": Detail},
+ })
def create_or_update_partition_policy(
response: Response,
request: Request,
@@ -244,9 +265,10 @@ def create_or_update_partition_policy(
audit_msg = f"user: {auth_data.user_id} xuserid: {auth_data.x_user_id} sha1: {sha1} requestid: {x_request_id} traceid: {x_b3_traceid} spanid: {x_b3_spanid} parentspanid: {x_b3_parentspanid} forward: {x_forwarded_for}"
+ validated = False
# Verify if ENABLE_VERIFY_POLICY is set and not Mocking OPA
- if conf.ENABLE_VERIFY_POLICY and not cloud_provider in (None, conf.MOCK):
- verify_policy_with_opa(policy_id=policy_id, auth_data=auth_data, correlation_id=context["correlation_id"], data=contents.decode('utf-8'))
+ if conf.ENABLE_VERIFY_POLICY and cloud_provider not in (None, conf.MOCK):
+ validated = verify_policy_with_opa(policy_id=policy_id, auth_data=auth_data, correlation_id=context["correlation_id"], data=contents.decode('utf-8'))
if cloud_provider is None:
# Help support MOCK testing
@@ -277,6 +299,7 @@ def create_or_update_partition_policy(
"data_partition": data_partition,
"opa_payload": contents,
"status_code": result.status_code,
+ "validated": validated,
"sha1": sha1,
"status": result.ok,
"message": result.message}
@@ -285,5 +308,5 @@ def create_or_update_partition_policy(
logger.error(f"Error when talking to OPA {result.message}")
raise HTTPException(status_code=result.status_code, detail=f"Error when talking to OPA {result.message}")
else:
- logger.error(f"Error when talking to bundle service {result.message}. {audit_msg}")
+ logger.error(f"Error when talking to bundle service {result.status_code} {result.message}. {audit_msg}")
raise HTTPException(status_code=result.status_code, detail=f"Error when talking to bundle service {result.message}")
diff --git a/app/api/tenant_api.py b/app/api/tenant_api.py
index 7248317b8a9fe38cb82951eae854597af12cbed2..f22676db6edbe36fb41a0d49c338ae509c60c288 100644
--- a/app/api/tenant_api.py
+++ b/app/api/tenant_api.py
@@ -3,27 +3,33 @@ from fastapi import Depends, HTTPException, Response
from auth import auth
import logging
import conf
-import json
from starlette_context import context
import k8s
-router = fastapi.APIRouter()
import correlation
import ruamel.yaml
from kubernetes.client.models import V1ConfigMap, V1ObjectMeta
+from pydantic import BaseModel
from starlette.status import (
- HTTP_201_CREATED,
HTTP_202_ACCEPTED,
+ HTTP_400_BAD_REQUEST,
HTTP_405_METHOD_NOT_ALLOWED,
HTTP_404_NOT_FOUND,
- HTTP_424_FAILED_DEPENDENCY,
HTTP_501_NOT_IMPLEMENTED,
HTTP_502_BAD_GATEWAY
)
+router = fastapi.APIRouter()
+
+class Detail(BaseModel):
+ detail: str
-logger = logging.getLogger(__name__)
+#logger = logging.getLogger(__name__)
-@router.get("/tenant")
+@router.get("/tenant", responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_405_METHOD_NOT_ALLOWED: {"model": Detail},
+ HTTP_501_NOT_IMPLEMENTED: {"model": Detail},
+ })
def get_tenant(response: Response,
all_data: bool = False,
auth_data: auth.Auth = Depends(auth.require_authorized_admin)
@@ -49,6 +55,7 @@ def get_tenant(response: Response,
else:
raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=f"{path} not found in OPA configmap.")
else:
+ logger.info("GET /tenant HTTP_405_METHOD_NOT_ALLOWED")
raise HTTPException(status_code=HTTP_405_METHOD_NOT_ALLOWED, detail=f"Unable to read configmap '{conf.OPA_CONFIG_MAP}' details")
def _mlput(self, path, value, index=None, sep=':'):
@@ -59,7 +66,12 @@ def _mlput(self, path, value, index=None, sep=':'):
else:
parent.insert(index, spath[-1], value)
-@router.put("/tenant")
+@router.put("/tenant", responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_405_METHOD_NOT_ALLOWED: {"model": Detail},
+ HTTP_501_NOT_IMPLEMENTED: {"model": Detail},
+ HTTP_502_BAD_GATEWAY: {"model": Detail},
+ })
def update_tenant(response: Response,
service: str,
polling_min_delay_seconds: int = 6,
@@ -69,6 +81,10 @@ def update_tenant(response: Response,
"""
Experimental tenant API for updating OPA bundle config for a data partition.
Adding new partitions is not supported in M20.
+
+ Services must be one of the supported (s3, gcs, gcp, blob, ...). See documentation for full list.
+ Min polling 1 second
+ Max polling 3600(1 hour)
"""
logging.setLogRecordFactory(correlation.set_correlation_id(context["correlation_id"]))
logger = logging.getLogger(__name__)
@@ -76,6 +92,12 @@ def update_tenant(response: Response,
logger.info(f"service: {service}, polling_min_delay_seconds: {polling_min_delay_seconds}, polling_max_delay_seconds: {polling_max_delay_seconds}")
+ if service not in conf.OPA_SUPPORTED_SERVICES:
+ raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail="Unsupported service provided")
+
+ if polling_min_delay_seconds >= polling_max_delay_seconds or polling_min_delay_seconds < 1 or polling_max_delay_seconds > conf.OPA_MAX_POLLING:
+ raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail="Unsupported polling_min_delay_seconds polling_max_delay_seconds")
+
config_map = k8s.get_opa_config(namespace=conf.NAMESPACE, name=conf.OPA_CONFIG_MAP, config_only=False)
if config_map:
path = f"osdu/partition/{auth_data.data_partition_id}"
@@ -126,9 +148,14 @@ def update_tenant(response: Response,
raise HTTPException(status_code=HTTP_501_NOT_IMPLEMENTED, detail=f"{path} not found in {data}")
else:
- raise HTTPException(status_code=HTTP_405_METHOD_NOT_ALLOWED, detail=f"Unable to read configmap details for {path}")
+ logger.info("PUT /tenant HTTP_405_METHOD_NOT_ALLOWED")
+ raise HTTPException(status_code=HTTP_405_METHOD_NOT_ALLOWED, detail="Unable to read configmap details")
-@router.delete("/tenant")
+@router.delete("/tenant", responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_405_METHOD_NOT_ALLOWED: {"model": Detail},
+ HTTP_501_NOT_IMPLEMENTED: {"model": Detail},
+ })
def delete_tenant(response: Response,
auth_data: auth.Auth = Depends(auth.require_authorized_admin),
):
@@ -148,4 +175,5 @@ def delete_tenant(response: Response,
del data["bundles"][path]
raise HTTPException(status_code=HTTP_501_NOT_IMPLEMENTED, detail=f"{path} {data}")
else:
- raise HTTPException(status_code=HTTP_405_METHOD_NOT_ALLOWED, detail=f"Unable to read configmap details for {path}")
\ No newline at end of file
+ logger.info("DELETE /tenant HTTP_405_METHOD_NOT_ALLOWED")
+ raise HTTPException(status_code=HTTP_405_METHOD_NOT_ALLOWED, detail="Unable to read configmap details")
\ No newline at end of file
diff --git a/app/api/translate_api.py b/app/api/translate_api.py
index 962b2ba7e98746ae3bc177890b9f06bf8cbedebf..592ef0af5f3beae5ea2fdf515bde358d3f2935f0 100644
--- a/app/api/translate_api.py
+++ b/app/api/translate_api.py
@@ -1,15 +1,14 @@
-from email import policy
+#from email import policy
import fastapi
from fastapi import HTTPException, Depends, Response
-import requests
import json
import logging
import re
-router = fastapi.APIRouter()
import opa
import conf
from starlette_context import context
import correlation
+from pydantic import BaseModel
from starlette.status import (
HTTP_500_INTERNAL_SERVER_ERROR,
@@ -24,9 +23,21 @@ from auth import auth
import translate.convert
from translate.translate_exceptions import PolicyDenyError, PolicyNotFoundError
+class Detail(BaseModel):
+ detail: str
+
+router = fastapi.APIRouter()
logger = logging.getLogger(__name__)
-@router.post("/translate", tags=["translate"])
+@router.post("/translate",
+ tags=["translate"],
+ responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_404_NOT_FOUND: {"model": Detail},
+ HTTP_406_NOT_ACCEPTABLE: {"model": Detail},
+ HTTP_422_UNPROCESSABLE_ENTITY: {"model": Detail},
+ HTTP_500_INTERNAL_SERVER_ERROR: {"model": Detail},
+ })
def translate_policy_api(response: Response, trans: TranslateItem, auth_data: auth.Auth = Depends(auth.require_authorized_user)):
"""
## Translate policy
@@ -42,8 +53,8 @@ def translate_policy_api(response: Response, trans: TranslateItem, auth_data:
posted_data = json.loads(trans.tojson()) # convert to dict for easy manipulation
logger.debug(f"data: {posted_data}")
- if not 'input' in posted_data:
- raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail=f"Input not found in data")
+ if 'input' not in posted_data:
+ raise HTTPException(status_code=HTTP_422_UNPROCESSABLE_ENTITY, detail="Input not found in data")
posted_data["input"]["token"] = auth_data.access_token
posted_data["input"]["xuserid"] = auth_data.x_user_id
@@ -162,10 +173,10 @@ def translate_preprocess(posted_data, data_partition):
elif original_query.startswith('data.osdu.instance.'):
policy_id = original_query.split('.')[3]
else:
- raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=f"Translate API Error: expected query in format data.osdu.partition...")
+ raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail="Translate API Error: expected query in format data.osdu.partition...")
path = conf.PARTITION_BUNDLE_ROOT.format(data_partition) + "/{0}".format(policy_id)
else:
- raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=f"Translate API Error: expected query in request")
+ raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail="Translate API Error: expected query in request")
logger.debug(f"PolicyID: {policy_id} DATA_PARTITION: {data_partition} PATH: {path}")
opa_response = opa.data(query=json.dumps(posted_data), path=path, name="data api for preprocess translate")
diff --git a/app/api/validate_api.py b/app/api/validate_api.py
index 7d4bb514960480401e02af352725c3e24d0902ca..183e21b982abee3c2f9317b807a7a9f878ffd58c 100644
--- a/app/api/validate_api.py
+++ b/app/api/validate_api.py
@@ -1,26 +1,30 @@
-from fastapi import APIRouter, Depends, File, UploadFile, HTTPException, Query, Response, Request
+from fastapi import APIRouter, Depends, UploadFile, HTTPException, Response, Request
from starlette_context import context
-import os
import logging
from requests.structures import CaseInsensitiveDict
from auth import auth
-import json
-import requests
-router = APIRouter()
-import conf
-import _buildinfo as b
import opa
import correlation
from bundles import bundle
from string import Template
import hashlib
+from pydantic import BaseModel
+
from starlette.status import (
+ HTTP_400_BAD_REQUEST,
HTTP_422_UNPROCESSABLE_ENTITY,
)
+router = APIRouter()
logger = logging.getLogger(__name__)
-@router.put("/validate/{policy_id}")
+class Detail(BaseModel):
+ detail: str
+
+@router.put("/validate/{policy_id}", responses={
+ HTTP_400_BAD_REQUEST: {"model": Detail},
+ HTTP_422_UNPROCESSABLE_ENTITY: {"model": Detail},
+ })
def validate_policy(
response: Response,
request: Request,
@@ -68,6 +72,10 @@ def validate_policy(
sha1=hashlib.sha1(data.encode()).hexdigest()
response.headers["X-SHA-1"] = sha1
return f"{policy_id} valid as osdu.partition[\"{auth_data.data_partition_id}\"].{short_name} {sha1}"
+ else:
+ # Not expected to get to this code block
+ logger.error(f"Invalid {policy_id}, unexpected return from OPA")
+ raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail="Invalid request")
def verify_policy_with_opa(policy_id: str, auth_data: auth.Auth, data: str, correlation_id: str):
@@ -84,9 +92,9 @@ def verify_policy_with_opa(policy_id: str, auth_data: auth.Auth, data: str, corr
headers["data-partition-id"] = data_partition
headers["Content-Type"] = "application/octet-stream"
- data = data.replace(f"package osdu.partition[", f"package tmp.osdu.partition[")
+ data = data.replace("package osdu.partition[", "package tmp.osdu.partition[")
- temp_id = f'tmp/{context["correlation_id"]}/{data_partition}/{policy_id}'
+ temp_id = f'tmp/{data_partition}/{policy_id}'
logger.info(f"TMP-AUDIT-OPA put id: {temp_id} user: {auth_data.user_id}")
temp_result = opa.put_opa_policy_direct(policy_id=temp_id, data=data, headers=headers, validate=True)
diff --git a/app/auth/auth.py b/app/auth/auth.py
index e3a490c363c170a07615a01717f33e2710ef95b0..f169777612ecac908e9d926196db5025e7765ad1 100644
--- a/app/auth/auth.py
+++ b/app/auth/auth.py
@@ -1,25 +1,18 @@
-from os import access
from fastapi import Depends, Request, Header, HTTPException
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
-from requests import head
from starlette.authentication import AuthCredentials
from requests.structures import CaseInsensitiveDict
-import requests
import logging
-import json
import conf
from starlette_context import context
from entitlement import EntitlementService
+import re
import correlation
from starlette.status import (
- HTTP_200_OK,
- HTTP_201_CREATED,
HTTP_400_BAD_REQUEST,
HTTP_401_UNAUTHORIZED,
HTTP_403_FORBIDDEN,
- HTTP_404_NOT_FOUND,
- HTTP_422_UNPROCESSABLE_ENTITY,
)
_logger = logging.getLogger(__name__)
@@ -41,7 +34,7 @@ async def require_data_partition_id(
if data_partition_id is None:
raise HTTPException(
status_code=HTTP_400_BAD_REQUEST,
- detail=f"Bad Request. Missing data_partition_id in header",
+ detail="Bad Request. Missing data_partition_id in header",
headers={"WWW-Authenticate": "Bearer"},
)
return data_partition_id
@@ -72,6 +65,11 @@ async def require_authorized_user(
token = credentials.credentials.strip()
+ # handle junk in data_partition_id
+ data_partition_id = str(data_partition_id)
+ if not re.match(pattern=conf.ALLOW_DATA_PARTITION_ID_PATTERN, string=data_partition_id):
+ raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail="Unsupported data-partition-id")
+
auth_data = Auth(access_token=token, data_partition_id=data_partition_id, x_user_id=x_user_id)
headers = CaseInsensitiveDict()
headers["Authorization"] = "Bearer " + token
@@ -113,7 +111,7 @@ async def require_authorized_admin(request: Request, auth_data: Auth = Depends(r
if conf.ADMIN_PERMISSION not in auth_data.groups:
raise HTTPException(
status_code=HTTP_403_FORBIDDEN,
- detail=f"User does not have the permission to call the Admin API",
+ detail="User does not have the permission to call the Admin API",
headers={"WWW-Authenticate": "Bearer"},
)
return auth_data
diff --git a/app/bundles/bundle.py b/app/bundles/bundle.py
index 159e340bb72b274f08ceb00b4965e69902f88cee..15af1752f44d9f5bbbf325130e3194818d382855 100644
--- a/app/bundles/bundle.py
+++ b/app/bundles/bundle.py
@@ -1,31 +1,19 @@
import logging
-from os import environ
import tarfile
import io
import time
-from fastapi import HTTPException
import hashlib
from starlette.status import (
HTTP_200_OK,
- HTTP_201_CREATED,
HTTP_202_ACCEPTED,
- HTTP_400_BAD_REQUEST,
- HTTP_401_UNAUTHORIZED,
- HTTP_403_FORBIDDEN,
HTTP_404_NOT_FOUND,
- HTTP_422_UNPROCESSABLE_ENTITY,
- HTTP_500_INTERNAL_SERVER_ERROR,
- HTTP_502_BAD_GATEWAY,
HTTP_503_SERVICE_UNAVAILABLE
)
-import conf
from bundles import storage
-from osdu_api.providers.types import FileLikeObject
from opa_response import OpaResponse
-import correlation
logger = logging.getLogger(__name__)
#logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": "None"})
@@ -93,7 +81,7 @@ def get_bundle(data_partition):
# logger.critical(f"Error Downloading {bundle} from CSP Storage")
except Exception as err:
logger.critical(f"Error Downloading {bundle} from CSP Storage: {err}")
- status_code = HTTP_503_SERVICE_UNAVAILABLE
+ #status_code = HTTP_503_SERVICE_UNAVAILABLE
message = f"Unable to get bundle from bundle server: download bundle {bundle} error: {err}"
#template = "An exception of type {0} occurred. Arguments:\n{1!r}"
#message = template.format(type(err).__name__, err.args)
@@ -125,7 +113,7 @@ def put_default_bundle(data_partition, policy_list):
logger.critical(f"Error uploading {bundle} to CSP Storage")
except Exception as err:
logger.critical(f"Error uploading {bundle} from CSP Storage: {err}")
- status_code = HTTP_503_SERVICE_UNAVAILABLE
+ #status_code = HTTP_503_SERVICE_UNAVAILABLE
message = f"Unable to upload bundle {bundle} to bundle server error: {err}"
#template = "An exception of type {0} occurred. Arguments:\n{1!r}"
#message = template.format(type(err).__name__, err.args)
@@ -196,6 +184,7 @@ def put_policy(data_partition, policy_id, policy):
response.status_code = HTTP_202_ACCEPTED
return response
else:
+ response.status_code = HTTP_503_SERVICE_UNAVAILABLE
response.message = f"Unable to add policy {policy_id} to bundle server: upload error {bundle}"
logger.critical(f"Error uploading {bundle} to CSP Storage")
diff --git a/app/bundles/providers/aws/storage.py b/app/bundles/providers/aws/storage.py
index 6ce15613989d25d9ce346f61c4cd2ad6fefc10f3..9ab115f29601b9174be56f750319a156aacc0b3d 100644
--- a/app/bundles/providers/aws/storage.py
+++ b/app/bundles/providers/aws/storage.py
@@ -20,11 +20,9 @@ from osdu_api.providers.types import FileLikeObject
import boto3
from bundles.storage import BundleStorageClient
-import correlation
import conf
logger = logging.getLogger(__name__)
-#logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": "None"})
logger.setLevel(conf.LOG_LEVEL)
class FileNotFound(Exception):
@@ -57,7 +55,7 @@ class AWSBundleStorageClient(BundleStorageClient):
raise FileNotFound(filename)
except Exception as e:
- logger.error(f"Failed to download file from {uri} {e}")
+ logger.error(f"Failed to download file from {uri}: {e}")
return None, None
def upload_file(self, name: str, file: FileLikeObject) -> str:
@@ -66,7 +64,7 @@ class AWSBundleStorageClient(BundleStorageClient):
self.client.upload_file(uri, file, self.content_type)
return uri
except Exception as e:
- logger.error(f"Failed to upload file to {uri} {e}")
+ logger.error(f"Failed to upload file to {uri}: {e}")
# TODO: Fix bug in AWS python SDK and replace
def _does_file_exist(self, uri: str) -> bool:
@@ -86,7 +84,7 @@ class AWSBundleStorageClient(BundleStorageClient):
# fast operation no matter the size of the data object
self.s3_client.head_object(Bucket=bucket_name, Key=object_name)
except Exception as e:
- logger.info(f"{uri} does not exist")
+ logger.info(f"{uri} does not exist {e}")
return False
return True
diff --git a/app/bundles/providers/azure/storage.py b/app/bundles/providers/azure/storage.py
index f2c2cd0c1216ed42ee41d0ca8f5f9ed2aacda429..9283cfec1dfabe85019aaa4bfc94309bb16c5d7e 100644
--- a/app/bundles/providers/azure/storage.py
+++ b/app/bundles/providers/azure/storage.py
@@ -4,7 +4,8 @@ import os
from typing import Tuple
from osdu_api.providers.types import FileLikeObject
from bundles.storage import BundleStorageClient
-from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
+#from azure.storage.blob import BlobServiceClient, BlobClient, ContainerClient
+from azure.storage.blob import BlobServiceClient
from azure.identity import DefaultAzureCredential
from azure.keyvault.secrets import SecretClient
@@ -30,7 +31,7 @@ class AzureBundleStorageClient(BundleStorageClient):
downloader.readinto(file)
return file, uri
except Exception as e:
- logger.error(f"Failed to download file from {uri}")
+ logger.error(f"Failed to download file from {uri}: {e}")
def upload_file(self, name: str, file: FileLikeObject) -> str:
try:
@@ -40,7 +41,7 @@ class AzureBundleStorageClient(BundleStorageClient):
blob_client.upload_blob(file.read(), overwrite=True, blob_type="BlockBlob")
return uri
except Exception as e:
- logger.error(f"Failed to upload file to {uri}")
+ logger.error(f"Failed to upload file to {uri}: {e}")
class Helper(object):
diff --git a/app/bundles/providers/ibm/storage.py b/app/bundles/providers/ibm/storage.py
index b1d2b44a33c708bf60bcde0ce910eb0feb5738eb..9ed473b6e64378e4eeec247a82771e0a0e86afd3 100644
--- a/app/bundles/providers/ibm/storage.py
+++ b/app/bundles/providers/ibm/storage.py
@@ -3,7 +3,7 @@ import os
from typing import Tuple
import boto3
from botocore.client import Config
-from osdu_api.providers.blob_storage import get_client
+#from osdu_api.providers.blob_storage import get_client
from osdu_api.providers.types import FileLikeObject
from bundles.storage import BundleStorageClient
diff --git a/app/bundles/storage.py b/app/bundles/storage.py
index 61d2b5e61465b063f13bcbc5c18cce4a0e5a6c7d..65dd1e817a5bd7fecb00702dcd6bb0a619fa2a14 100644
--- a/app/bundles/storage.py
+++ b/app/bundles/storage.py
@@ -21,8 +21,6 @@ import sys
from typing import Tuple
from osdu_api.providers.types import FileLikeObject
-import correlation
-import conf
logger = logging.getLogger(__name__)
#logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": "None"})
diff --git a/app/conf.py b/app/conf.py
index 06a164f795aabd1ae6c2047eaac2fdf20ffa71fe..c4e1a5919dbaa1ecc6e1b1728f86061b4e479e9a 100644
--- a/app/conf.py
+++ b/app/conf.py
@@ -139,5 +139,12 @@ if os.path.isfile("/var/run/secrets/kubernetes.io/serviceaccount/token"):
TOKEN = f.read()
OPA_CONFIG_MAP = os.getenv("OPA_CONFIG_MAP", 'opa-config')
+OPA_MAX_POLLING = 3600
+
+ALLOW_DATA_PARTITION_ID_PATTERN = "^[A-Za-z0-9_-]*$"
+ALLOW_CORRELATION_ID_PATTERN = "^[A-Za-z0-9_-]*$"
+
+# OCI Registry not supported
+OPA_SUPPORTED_SERVICES = ['s3', 'gcs', 'gcp', 'blob', 'nginx']
CLOUD_PROVIDER = os.getenv("CLOUD_PROVIDER", 'LOCAL')
diff --git a/app/correlation.py b/app/correlation.py
index 2f019bfb2d1f13e579b0a64a125cd9a0bd947255..e5b20e98f669adcdfd087c61a05a0e54bae65c8b 100644
--- a/app/correlation.py
+++ b/app/correlation.py
@@ -1,8 +1,14 @@
import logging
from fastapi import Header
-from starlette_context import context, request_cycle_context
+from fastapi import HTTPException
+from starlette_context import request_cycle_context
from uuid_extensions import uuid7str
-from conf import GENERATE_CORRELATION_ID, LOG_LEVEL
+from conf import GENERATE_CORRELATION_ID, LOG_LEVEL, ALLOW_CORRELATION_ID_PATTERN
+import re
+
+from starlette.status import (
+ HTTP_400_BAD_REQUEST,
+)
#import coloredlogs
#class CorrelationIDLoggerAdapter(logging.LoggerAdapter):
@@ -40,6 +46,9 @@ async def policy_context_dependency(correlation_id: str = Header(None), user_age
if not correlation_id and GENERATE_CORRELATION_ID:
correlation_id = uuid7str()
+ else:
+ if not re.match(pattern=ALLOW_CORRELATION_ID_PATTERN, string=correlation_id):
+ raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail="Unsupported correlation-id")
data = {"correlation_id": correlation_id,
"user_agent": user_agent
diff --git a/app/entitlement.py b/app/entitlement.py
index 9603b83bc93fb04222597eede0237bb21e1ab6df..3b7b474e0d8963ca7b650ad2061f348c4149864b 100644
--- a/app/entitlement.py
+++ b/app/entitlement.py
@@ -1,10 +1,10 @@
# entitlement.py
#from asyncio.log import logger
import logging
-from email import header
+#from email import header
from fastapi import HTTPException
import requests
-from starlette.status import HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN
+from starlette.status import HTTP_401_UNAUTHORIZED
from starlette_context import context
import json
import conf
@@ -51,6 +51,6 @@ class EntitlementService:
# this should never happen
raise HTTPException(
status_code=HTTP_401_UNAUTHORIZED,
- detail=f"Authentication was unsuccessful. Groups undefined",
+ detail="Authentication was unsuccessful. Groups undefined",
headers={"WWW-Authenticate": "Bearer"},
)
\ No newline at end of file
diff --git a/app/k8s.py b/app/k8s.py
index e10bf1fa1edd1f6aad0a02a9f204e8376bfa78f0..732be7b1ac7aab263d0fe181ed423f4516105b77 100755
--- a/app/k8s.py
+++ b/app/k8s.py
@@ -27,6 +27,9 @@ def get_opa_config(namespace="osdu-services", name="opa-config", config_only=Tru
except config.config_exception.ConfigException as err:
logger.error(f"Exception when calling kubernetes load config {conf.CLOUD_PROVIDER}: {err}")
return None
+ except Exception as e:
+ logger.error(f"Unknown exception {e} when calling kubernetes load config {conf.CLOUD_PROVIDER}")
+ return None
v1 = client.CoreV1Api()
@@ -47,6 +50,8 @@ def get_opa_config(namespace="osdu-services", name="opa-config", config_only=Tru
except ApiException as err:
# If you get 403 errors from the API server you will have to configure K8s RBAC to add permission
logger.error(f"Exception when calling CoreV1Api->list_namespaced_config_map: {err}")
+ except Exception as e:
+ logger.error(f"Unknown exception {e} when calling CoreV1Api->list_namespaced_config_map")
return None
def patch_opa_config(namespace="osdu-services", name="opa-config", pretty=True, body=None):
@@ -83,7 +88,7 @@ if __name__ == '__main__':
data_partition = 'osdu'
name = "opa-config"
r = get_opa_config(name=name, namespace=conf.NAMESPACE)
- if f"osdu/partition/{data_partition}" in r and f"bundle-{data_partition}.tar.gz" in r:
+ if r and f"osdu/partition/{data_partition}" in r and f"bundle-{data_partition}.tar.gz" in r:
print(f"found '{data_partition}' in configmap {name} in {conf.NAMESPACE}")
else:
print(f"not found '{data_partition}' in configmap {name} in {conf.NAMESPACE}")
\ No newline at end of file
diff --git a/app/main.py b/app/main.py
index 490cd6c7fd2c481c34e1367bf5ec4af4d170f94d..d6b2cfb3156e83e560c871d0f31601ef9b8a6571 100644
--- a/app/main.py
+++ b/app/main.py
@@ -1,12 +1,12 @@
#!/usr/bin/env python3
# OSDU Policy Service
-from fastapi import FastAPI, Request, Depends, BackgroundTasks, Header, Depends, Response
-from fastapi.responses import RedirectResponse, HTMLResponse
-from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm, HTTPBearer
-from fastapi.openapi.docs import get_swagger_ui_html
-from fastapi.openapi.utils import get_openapi
+from fastapi import FastAPI, Request, Depends
+#from fastapi.responses import RedirectResponse, HTMLResponse
+#from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm, HTTPBearer
+#from fastapi.openapi.docs import get_swagger_ui_html
+#from fastapi.openapi.utils import get_openapi
from fastapi.staticfiles import StaticFiles
-from starlette_context import context, request_cycle_context
+#from starlette_context import context, request_cycle_context
import uvicorn
import os
import logging
@@ -14,7 +14,7 @@ import conf
from api import policy_read_api, policy_update_api, policy_eval_api, health_api, compile_api, info_api, translate_api, diag_api, login_api, backup_api, bootstrap_api, config_api, tenant_api, validate_api
#from models.translate import TranslateItem
from views import home
-import _buildinfo as b
+import _version
from auth import auth
#from fastapi.middleware.cors import CORSMiddleware
from starlette.middleware import Middleware
@@ -61,7 +61,7 @@ tags_metadata = [
]
description = f"""
- OSDU Policy Service v{b.version} API for Milestone {b.milestone}
+ OSDU Policy Service v{_version.__version__} API for {_version.__milestone__}
Policy service is used for management and evaluation of dynamic policies in OSDU.
@@ -141,7 +141,7 @@ app = FastAPI(
middleware=middleware,
title="OSDU Policy Service",
description=description,
- version=f"v{b.version} {b.milestone}",
+ version=f"v{_version.__version__} {_version.__milestone__}",
docs_url=conf.SERVICE_BASE_PATH + "/docs",
redoc_url=conf.SERVICE_BASE_PATH + "/redoc",
openapi_url=conf.SERVICE_BASE_PATH + "/openapi.json",
@@ -256,12 +256,12 @@ def configure_routing():
def configure_middleware():
# This will need some work before Admin UI is moved out of POC
- origins = [
- "http://localhost",
- "http://localhost:8080",
- "http://0.0.0.0:8080",
- "http://localhost:4200"
- ]
+ # origins = [
+ # "http://localhost",
+ # "http://localhost:8080",
+ # "http://0.0.0.0:8080",
+ # "http://localhost:4200"
+ # ]
app.add_middleware(
CORSMiddleware,
allow_origins=['*'],
diff --git a/app/opa.py b/app/opa.py
index f2fe6e667f0a3f58f59917fad78f1c91c833d1d4..9b60411583279ec58afa33cf11d9fc79515fe190 100644
--- a/app/opa.py
+++ b/app/opa.py
@@ -1,4 +1,3 @@
-from urllib import response
import conf
import requests
import logging
@@ -7,6 +6,7 @@ from starlette_context import context
from opa_response import OpaResponse
import correlation
from cachetools import cached, TTLCache
+import requests.exceptions
#import httpx
import timer
@@ -55,6 +55,8 @@ def fetch_opa_policy_direct(policy_id, timeout=20, name="fetch_opa_policy"):
def fetch_opa_policy_direct_with_caching(policy_id, timeout=20):
"""
fetch a specific policy in OPA
+ 200 - no error
+ 500 - server error
"""
logging.setLogRecordFactory(correlation.set_correlation_id(context["correlation_id"]))
logger = logging.getLogger(__name__)
@@ -122,7 +124,7 @@ def put_opa_policy_direct(policy_id, data, timeout=20, headers=None, validate=Fa
logger = logging.getLogger(__name__)
url = conf.OPA_POLICIES_API + '/' + policy_id
logger.info(f"OPA put URL {url}")
- result = OpaResponse()
+ result = OpaResponse(status_code=503)
if not headers:
headers = headers={'Content-Type': 'application/octet-stream'}
@@ -141,9 +143,12 @@ def put_opa_policy_direct(policy_id, data, timeout=20, headers=None, validate=Fa
else:
logger.error("Error updating policy with OPA direct, got status %s and message: %s" % (rsp.status_code, rsp.text))
result.message = f"{rsp.text} {rsp.status_code}"
+ except requests.ConnectionError as err:
+ logger.error(f"Unexpected ConnectionError when attempting to connect to OPA: {err}")
+ result.message = "Unexpected ConnectionError when connecting to OPA"
except Exception as err:
logger.error(f"Unexpected error when attempting to connect to OPA: {err}")
- result.message = "Unexpected error"
+ result.message = "Unexpected error when connecting to OPA"
return result
def compile(query, metrics=False, instrument=False, timeout=20, name="compile api"):
@@ -196,7 +201,7 @@ def compile_with_caching(query, metrics=False, instrument=False, timeout=20):
if rsp.ok:
logger.debug(f"OK Response from OPA({url}): {json.dumps(result.json, indent=2)}")
- result.message = f"Policy OPA compiled"
+ result.message = "Policy OPA compiled"
else:
logger.error("Error compiling policy, got status %s and message: %s" % (rsp.status_code, rsp.text))
result.message = f"Error compiling policy: {rsp.text}"
@@ -250,7 +255,7 @@ def data_with_caching(query, path, timeout=20):
#if rsp.status_code == httpx.codes.OK:
if rsp.ok:
logger.debug(f"OK Response from OPA({url}): {json.dumps(result.json, indent=2)}")
- result.message = f"Policy OPA query"
+ result.message = "Policy OPA query"
else:
logger.error("Error compiling policy, got status %s and message: %s" % (rsp.status_code, rsp.text))
result.message = f"Error query policy: {rsp.text}"
@@ -284,11 +289,11 @@ def get_document_with_input_cached(rego_package_name, rego_rule_name, rego_input
try:
#async with httpx.AsyncClient() as client:
#response = await client.post(url,
- response = requests.post(url,
+ resp = requests.post(url,
data=json.dumps({"input": rego_input}), timeout=timeout)
except Exception as err:
logger.error(err)
- result = response.json()["result"]
+ result = resp.json()["result"]
return result
\ No newline at end of file
diff --git a/app/opa_response.py b/app/opa_response.py
index 8a7f9d3408cd345be9104e90c7c4c65678c77fbd..69af789638930da0faa396d34d57979dfa79a879 100644
--- a/app/opa_response.py
+++ b/app/opa_response.py
@@ -1,5 +1,4 @@
# opa_helper.py
-from unittest import result
class OpaResponse:
ok: bool
diff --git a/app/tests/integration/test_integration_010.py b/app/tests/integration/test_integration_010.py
index 8b3b96b58f9dd96fb059ac10562ae8a2cb58bd01..9587bd00f83498001ccee040bfe0c31cbbf9c4eb 100644
--- a/app/tests/integration/test_integration_010.py
+++ b/app/tests/integration/test_integration_010.py
@@ -45,7 +45,7 @@ def test_mock_health_api_endpoint_no_auth_opa_available(opa_access, cloud_provid
r = client.get(conf.SERVICE_BASE_PATH+"/health")
# If OPA is available should get a 200
# Some CSPs (like AWS) don't expose /health without auth so let's skip it in that case
- if r.status_code == 401:
+ if r.status_code == 401 or r.status_code == 403:
pytest.skip(f"Skipping mock /health unauth test for {cloud_provider}")
assert r.status_code == 200, f"Expect /health endpoint to be available {r.text}"
assert "Healthy" in r.text, "Expected Healthy in response text from /health"
@@ -56,7 +56,7 @@ def test_serviceurl_health_api_endpoint_no_auth_opa_available(service_url, cloud
r = requests.get(url)
# If OPA is available should get a 200
# Some CSPs (like AWS) don't expose /health without auth so let's skip it in that case
- if r.status_code == 401:
+ if r.status_code == 401 or r.status_code == 403:
pytest.skip(f"Skipping service_url /health unauth test for {cloud_provider}")
assert r.status_code == 200, f"Expect /health endpoint ({url}) to be available {r.text}"
assert "Healthy" in r.text, "Expected Healthy in response text from /health"
diff --git a/app/tests/integration/test_integration_011_utility.py b/app/tests/integration/test_integration_011_utility.py
index 2a40958c07da860058a5cf578daf4c2ef500df2b..a90b4af172e49ac9dc836c4e01b98aac152faa4e 100644
--- a/app/tests/integration/test_integration_011_utility.py
+++ b/app/tests/integration/test_integration_011_utility.py
@@ -57,5 +57,24 @@ def test_bootstrap(token, data_partition, service_url, cloud_provider):
url = service_url + conf.SERVICE_BASE_PATH + "/bootstrap"
headers={'Authorization': 'Bearer ' + token, 'data-partition-id': data_partition}
r = requests.post(url, headers=headers)
- if r.status_code != 201 or r.status_code != 202 or r.status != 405 or r.status_code != 424:
- pytest.xfail(f"service_url /bootstrap fail for {cloud_provider} {r.status_code} {r.text}")
\ No newline at end of file
+
+ if r.status_code == 424 or r.status_code == 405:
+ pytest.xfail(f"/bootstrap {cloud_provider} {r.status_code} {r.text}")
+
+ assert r.status_code == 201 or r.status_code == 202, f"/bootstrap {cloud_provider} {r.status_code} {r.text}"
+
+@pytest.mark.dependency(name="require_token")
+def test_bootstrap_force(token, data_partition, service_url, cloud_provider):
+ """
+ Test Bootstrap API Force
+ """
+ url = service_url + conf.SERVICE_BASE_PATH + "/bootstrap"
+ params = {'force': True}
+ headers={'Authorization': 'Bearer ' + token, 'data-partition-id': data_partition}
+ r = requests.post(url, headers=headers, params=params)
+
+ if r.status_code == 424 or r.status_code == 405:
+ pytest.xfail(f"/bootstrap {cloud_provider} {r.status_code} {r.text}")
+
+ assert r.status_code == 201 or r.status_code == 202, f"/bootstrap {cloud_provider} {r.status_code} {r.text}"
+
\ No newline at end of file
diff --git a/app/tests/integration/test_integration_012_validate.py b/app/tests/integration/test_integration_012_validate.py
index ffa317b9b51ca1247401b868ceec3a612324d15d..14614a3ceec04425f2c49deb12a2e79cb066adf4 100644
--- a/app/tests/integration/test_integration_012_validate.py
+++ b/app/tests/integration/test_integration_012_validate.py
@@ -61,3 +61,15 @@ records := centralauthz.records"""
files=files)
if r.status_code != 200:
pytest.xfail(f"service_url /validate fail for {cloud_provider}: {r.status_code} {r.text} {url}")
+
+@pytest.mark.parametrize("regofile", testlib.get_rego_templates())
+@pytest.mark.dependency(depends=["require_token"])
+def test_validate_policies(token, data_partition, service_url, domain, regofile):
+ assert token is not None, "No token provided on command line"
+ testlib.put_policy_test_data(client=requests,
+ token=token,
+ data_partition=data_partition,
+ service_url=service_url,
+ domain=domain,
+ filename=regofile,
+ validate=True)
\ No newline at end of file
diff --git a/app/tests/integration/test_integration_020_put.py b/app/tests/integration/test_integration_020_put.py
index 414fdadbaabb0eca4441ead6309058682c73e4d7..40f26b8b86f36818935f7fcd955559f4586c31fc 100644
--- a/app/tests/integration/test_integration_020_put.py
+++ b/app/tests/integration/test_integration_020_put.py
@@ -61,6 +61,8 @@ def get_rego_templatesx():
@pytest.mark.dependency(depends=["require_token"])
def test_put_policy(token, data_partition, service_url, domain, regofile):
assert token is not None, "No token provided on command line"
+ # Let's not burden the bundle service backend - rate limit issues
+ time.sleep(1)
testlib.put_policy_test_data(client=requests,
token=token,
data_partition=data_partition,
diff --git a/app/tests/integration/test_integration_035_eval.py b/app/tests/integration/test_integration_035_eval.py
index 793e083000bdbe7ade3e143f987ebd19441517c9..55b730cf4b5704ef92c6613c900a5f2de027d7dc 100644
--- a/app/tests/integration/test_integration_035_eval.py
+++ b/app/tests/integration/test_integration_035_eval.py
@@ -124,6 +124,7 @@ def eval_policy(token, data_partition, service_url, policy_name, expect=200, opa
# randomly select a legal tag
legal_tag = random.choice(get_legal_tags(token=token, data_partition=data_partition))
print(f"Using Legal tag: {legal_tag}")
+ print(f"Using policy_id: {policy_id}")
p = os.path.dirname(os.path.abspath(__file__))
template_datadir = os.path.join(p, '..', 'templates')
diff --git a/app/tests/integration/test_integration_040_delete.py b/app/tests/integration/test_integration_040_delete.py
index 7b3b4a623255747dd4807c581bb79116da0e6328..db43aef732a8d7ff06f8a0eb4b2885b8718522ce 100644
--- a/app/tests/integration/test_integration_040_delete.py
+++ b/app/tests/integration/test_integration_040_delete.py
@@ -42,11 +42,15 @@ def test_delete_osdu_partition_policies_service_url(token, data_partition, bundl
"""
Test delete /policies/osdu/partition/<data_partition>/<policy>
"""
- delete_skip_list = ['dataauthz.rego', 'search.rego', 'search2.rego']
+ delete_skip_list = ['dataauthz.rego', 'search.rego', 'search1.rego', 'search2.rego']
if regofile in delete_skip_list:
pytest.skip(f"skipping delete of {regofile}")
id = f"osdu/partition/{data_partition}/{regofile}"
url = conf.SERVICE_BASE_PATH + "/policies/" + id
+
+ # Let's not burden the bundle service backend - rate limit issues
+ time.sleep(1)
+
r = requests.delete(service_url + url,
headers={'Authorization': 'Bearer ' + token,
'data-partition-id': data_partition})
diff --git a/app/tests/integration/test_integration_045_delete.py b/app/tests/integration/test_integration_045_delete.py
index 70162b7390c6edb3032f38ce5ac6efdd0310e253..8b8e8fd35ec98f61f791297f495f700f7393d82b 100644
--- a/app/tests/integration/test_integration_045_delete.py
+++ b/app/tests/integration/test_integration_045_delete.py
@@ -10,6 +10,7 @@ import logging
import time
import sys
import os
+import time
import base64
import json
from pathlib import Path
@@ -44,7 +45,7 @@ def test_confirm_deletion_of_osdu_partition_policies_expect_not_found_service_ur
#pytest.skip("Skipping delete")
num_tests = 0
p = os.path.dirname(os.path.abspath(__file__))
- delete_skip_list = ['dataauthz.rego', 'search.rego', 'search2.rego']
+ delete_skip_list = ['dataauthz.rego', 'search.rego', 'search1.rego', 'search2.rego']
template_datadir = os.path.join(p, '..', 'templates')
for filename in sorted(os.listdir(template_datadir)):
filetype='.rego'
@@ -53,9 +54,13 @@ def test_confirm_deletion_of_osdu_partition_policies_expect_not_found_service_ur
continue
id = f"osdu/partition/{data_partition}/{filename}"
url = conf.SERVICE_BASE_PATH + "/policies/" + id
+ # Let's not burden the bundle service backend - rate limit issues
+ time.sleep(1)
r = requests.delete(service_url + url,
headers={'Authorization': 'Bearer ' + token,
'data-partition-id': data_partition})
+ if r.status_code == 503:
+ print(f"Unexpected response {r.status_code} {r.text} rate limit?")
assert r.status_code == 404, f"Expect 404_NOT_FOUND. Delete of {filename} {url} should already be deleted"
assert "not_found" in r.text, f"expect not found message {r.text}"
num_tests = num_tests + 1
diff --git a/app/tests/integration/test_integration_099_load.py b/app/tests/integration/test_integration_099_load.py
index 4c5e1f7c0afe98159c67fc315b735e883603c317..958b9f9ec4d461a27206b23188ebbd60f6d9d306 100644
--- a/app/tests/integration/test_integration_099_load.py
+++ b/app/tests/integration/test_integration_099_load.py
@@ -39,6 +39,8 @@ def test_require_token(token):
@pytest.mark.dependency(depends=["require_token"])
def test_load_policy_service_url(token, data_partition, service_url, domain, regofile):
assert token is not None, "No token provided on command line"
+ # Let's not burden the bundle service backend - rate limit issues
+ time.sleep(1)
testlib.put_policy_test_data(client=requests,
token=token,
data_partition=data_partition,
diff --git a/app/tests/templates/search.rego b/app/tests/templates/search.rego
index ed9136bb7a4ef18054f506ebc64874d608289c6b..5f096408208a3892572700046e8dfe199e343fc3 100644
--- a/app/tests/templates/search.rego
+++ b/app/tests/templates/search.rego
@@ -1,34 +1,21 @@
# METADATA
-# title: Search
+# title: search
# description: |
-# New M18 Functionality
-# Have search service use the preprocessor functionality when calling translate
-# This requires policy search_preprocessor to be loaded.
+# Default search policy
# related_resources:
# - ref: https://community.opengroup.org/osdu/platform/security-and-compliance/policy/-/blob/master/app/tests/templates/search.rego
-
package osdu.partition["${data_partition}"].search
-import data.osdu.partition["${data_partition}"].search_preprocessor
-
-default allow := false
-default deny := false
-
-preprocess_config := {
- "input_from_preprocessor": {
- "allow_groups": search_preprocessor.allow_groups,
- "deny_groups": search_preprocessor.deny_groups
- },
- "has_allow_rule": true,
- "has_deny_rule": true
-}
+default allow = false
-allow {
- input.operation == "view"
- input.record.acl.owners[_] == input.allow_groups[_]
+allow = true {
+ input.operation == "view"
+ # At least one user group needs to be in acl viewers
+ input.record.acl.viewers[_]==input.groups[_]
}
-deny {
- input.operation == "view"
- input.record.acl.owners[_] == input.deny_groups[_]
+allow = true {
+ input.operation == ["view", "create", "update", "delete", "purge"][_]
+ # At least one user group needs to be in acl owners
+ input.record.acl.owners[_]==input.groups[_]
}
diff --git a/app/tests/templates/search1.rego b/app/tests/templates/search1.rego
new file mode 100644
index 0000000000000000000000000000000000000000..c1b5a6126a9a554e02dd769d4ca716c12a8334db
--- /dev/null
+++ b/app/tests/templates/search1.rego
@@ -0,0 +1,34 @@
+# METADATA
+# title: Search
+# description: |
+# New M18 Functionality
+# Have search service use the preprocessor functionality when calling translate
+# This requires policy search_preprocessor to be loaded.
+# related_resources:
+# - ref: https://community.opengroup.org/osdu/platform/security-and-compliance/policy/-/blob/master/app/tests/templates/search1.rego
+
+package osdu.partition["${data_partition}"].search1
+
+import data.osdu.partition["${data_partition}"].search_preprocessor
+
+default allow := false
+default deny := false
+
+preprocess_config := {
+ "input_from_preprocessor": {
+ "allow_groups": search_preprocessor.allow_groups,
+ "deny_groups": search_preprocessor.deny_groups
+ },
+ "has_allow_rule": true,
+ "has_deny_rule": true
+}
+
+allow {
+ input.operation == "view"
+ input.record.acl.owners[_] == input.allow_groups[_]
+}
+
+deny {
+ input.operation == "view"
+ input.record.acl.owners[_] == input.deny_groups[_]
+}
diff --git a/app/tests/testlib.py b/app/tests/testlib.py
index d6487d4688809cff5c2a3b5252d330c99d200d66..280dda036dc7a5f9171d97327e9bd9393ffbc2ee 100644
--- a/app/tests/testlib.py
+++ b/app/tests/testlib.py
@@ -1,7 +1,6 @@
# testlib.py
import os
import requests
-import json
import re
from string import Template
import pytest
@@ -34,7 +33,8 @@ def put_policy_test_data(
filename,
data_partition_header=None,
service_url=None,
- domain="example.com"):
+ domain="example.com",
+ validate=False):
"""
This supporting function takes
client = FastAPI test client or requests
@@ -79,7 +79,7 @@ def put_policy_test_data(
'DOMAIN': domain
})
- if not (f"package osdu.partition[\"{data_partition}\"].{policy_id_short}" in data):
+ if f"package osdu.partition[\"{data_partition}\"].{policy_id_short}" not in data:
print(f"expected matching package name '{policy_id_short}' for {file_path} skipping")
pytest.xfail(f"expected matching package name '{policy_id_short}' for {file_path}")
@@ -87,7 +87,11 @@ def put_policy_test_data(
bdata = data.encode('utf-8')
files = {'file': (policy_id, bdata)}
- url = conf.SERVICE_BASE_PATH + "/policies/osdu/partition/" + data_partition + "/" + policy_id
+ if validate:
+ url = conf.SERVICE_BASE_PATH + "/validate/" + policy_id
+ else:
+ url = conf.SERVICE_BASE_PATH + "/policies/osdu/partition/" + data_partition + "/" + policy_id
+
if service_url:
url = service_url + url
r = requests.put(url,
@@ -99,15 +103,22 @@ def put_policy_test_data(
files = files,
headers={'Authorization': 'Bearer ' + token,
'data-partition-id': data_partition_header})
- assert r.status_code == 200 or r.status_code == 202, f"Expected to get 200 or 202, got {r.status_code} from {url} {r.text}"
- assert "added to " in r.text or "updated in" in r.text or "already in" in r.text, f"Expect 'added to... or updated in...' message in result but found {r.text}"
- assert policy_id in r.text, f"Expect {policy_id} message in result {r.text}"
- result = r.json()
- #print(f"returned json: {json.dumps(result, indent=4)}")
- assert result["status"] == True, "Expected status true (success)"
+ if validate:
+ if r.status_code == 422 or r.status_code == 400:
+ # Invalid rego. Let's just stop now
+ pytest.fail(f"{os.environ['PYTEST_CURRENT_TEST']} Failed validation! {r.status_code} {r.text}")
+ assert r.status_code == 200 or r.status_code == 202, f"Expected to get 200 or 202, got {r.status_code} from {url} {r.text}"
+ assert " valid " in r.text, f"Expect 'valid' in message of result but found {r.text}"
+ else:
+ assert r.status_code == 200 or r.status_code == 202, f"Expected to get 200 or 202, got {r.status_code} from {url} {r.text}"
+ assert "added to " in r.text or "updated in" in r.text or "already in" in r.text, f"Expect 'added to... or updated in...' message in result but found {r.text}"
+ assert policy_id in r.text, f"Expect {policy_id} message in result {r.text}"
+ result = r.json()
+ #print(f"returned json: {json.dumps(result, indent=4)}")
+ assert result["status"] is True, "Expected status true (success)"
num_tests = num_tests + 1
else:
- pytest.skip(f"Only .rego templates")
+ pytest.skip("Only .rego templates")
#assert num_tests >= 7, "At least 7 polices were tested"
def strip_comments(code):
diff --git a/app/tests/unit/test_api_unit.py b/app/tests/unit/test_api_unit.py
index aa4698bbeed2ce0145f65dba9e73516dcba29664..8658f54818738f2a4e1012f6029e6634c2f927f3 100644
--- a/app/tests/unit/test_api_unit.py
+++ b/app/tests/unit/test_api_unit.py
@@ -1,10 +1,6 @@
-from textwrap import indent
import pytest
import unittest
-import responses
from fastapi.testclient import TestClient
-import requests
-import re
import logging
import sys
import os
@@ -16,7 +12,6 @@ sys.path.append('tests')
# local
from main import app
-from auth import auth
import conf
import testlib
diff --git a/app/timer.py b/app/timer.py
index 7c59f87cd08bc951df5ee977f21e9776a1d16eb7..ed503cc8ae90c391bae5f9cd8a92dae2316d1d53 100644
--- a/app/timer.py
+++ b/app/timer.py
@@ -26,14 +26,14 @@ class Timer:
def start(self):
"""Start a new timer"""
if self._start_time is not None:
- raise TimerError(f"Timer is running. Use .stop() to stop it")
+ raise TimerError("Timer is running. Use .stop() to stop it")
self._start_time = time.perf_counter()
def stop(self):
"""Stop the timer, and report the elapsed time"""
if self._start_time is None:
- raise TimerError(f"Timer is not running. Use .start() to start it")
+ raise TimerError("Timer is not running. Use .start() to start it")
elapsed_time = time.perf_counter() - self._start_time
self._start_time = None
diff --git a/app/translate/convert.py b/app/translate/convert.py
index f66ff2dea4b4234c8f7e9d91bdbac41623754543..47aaa8553b1d25a7d81502e4e131a09035c8c172 100644
--- a/app/translate/convert.py
+++ b/app/translate/convert.py
@@ -15,12 +15,12 @@ from elasticsearch_dsl.query import (
Terms,
)
-from cachetools import cached, TTLCache
+#from cachetools import cached, TTLCache
import correlation
from . import ast
import opa
from .translate_exceptions import PolicyDenyError
-import conf
+#import conf
logger = logging.getLogger(__name__)
@@ -62,23 +62,23 @@ def _convert_opa_query_to_es_query(opa_query: ast.Query, opa_request_input):
logger.debug(f"performing side effect for {expr}")
if expr.op() == "http.send":
logger.debug(f"isistance {expr.terms[2].value}, {ast.Var}")
- assert isinstance(expr.terms[2].value, ast.Var), f"assert02 isinstance"
+ assert isinstance(expr.terms[2].value, ast.Var), "assert02 isinstance"
logger.debug("after assert")
http_result = _resolve_http(expr)
side_effect_context[expr.terms[2].value.value] = http_result
logger.debug(f"side effect context updated: {side_effect_context}")
elif expr.op() == "eq":
logger.debug(f"eq {expr.terms[1].value}, {str(ast.Ref)}")
- assert isinstance(expr.terms[1].value, ast.Ref), f"assert03 isinstance"
+ assert isinstance(expr.terms[1].value, ast.Ref), "assert03 isinstance"
logger.debug("after assert eq 1")
the_ref = expr.terms[1]
logger.debug(f"the_ref: {the_ref}")
logger.debug(f"assert: {the_ref.value.terms[0].value} {ast.Var}")
- assert isinstance(the_ref.value.terms[0].value, ast.Var), f"assert04 isinstance"
+ assert isinstance(the_ref.value.terms[0].value, ast.Var), "assert04 isinstance"
logger.debug("after assert eq 2")
variable_key = the_ref.value.terms[0].value.value
logger.debug(f"variable_key: {variable_key}")
- assert isinstance(the_ref.value.terms[1].value, ast.Scalar), f"assert05 isinstance"
+ assert isinstance(the_ref.value.terms[1].value, ast.Scalar), "assert05 isinstance"
logger.debug("after assert eq 3")
variable_path = the_ref.value.terms[1].value.value
logger.debug(f"variable_path: {variable_path}")
@@ -112,10 +112,10 @@ def _convert_opa_query_to_es_query(opa_query: ast.Query, opa_request_input):
scalar = operand.value
field_value = scalar.value
elif isinstance(operand.value, ast.Object):
- assert False, f"assert06 False"
+ assert False, "assert06 False"
logger.debug(operand.value)
elif isinstance(operand.value, ast.Var):
- assert False, f"assert07 False"
+ assert False, "assert07 False"
logger.debug(operand.value)
else:
not_scalar = operand.value
@@ -124,8 +124,8 @@ def _convert_opa_query_to_es_query(opa_query: ast.Query, opa_request_input):
for term in not_scalar.terms[2:]
if not isinstance(term.value, ast.Var)
)
- assert field_name is not None, f"assert08 field_name"
- assert field_value is not None, f"assert09 field_value"
+ assert field_name is not None, "assert08 field_name"
+ assert field_value is not None, "assert09 field_value"
if expr.op() == "eq":
es_filter = Term(**{field_name: field_value})
@@ -154,8 +154,8 @@ def _convert_opa_query_to_es_query(opa_query: ast.Query, opa_request_input):
)(expr.terms.value)
# Validate domain is input.record.xxx
- assert domain.value.operand(0).value.value == "input", f"assert10 input"
- assert domain.value.operand(1).value.value == "record", f"assert11 record"
+ assert domain.value.operand(0).value.value == "input", "assert10 input"
+ assert domain.value.operand(1).value.value == "record", "assert11 record"
# Support only simple body
assert len(body) == 1, f"assert12 body len {body}"
@@ -163,10 +163,10 @@ def _convert_opa_query_to_es_query(opa_query: ast.Query, opa_request_input):
# Ensure body is negated
# ElasticSearch, being an inverted index, cannot support universal quantification
- assert body.negated, f"assert13 body negated"
+ assert body.negated, "assert13 body negated"
# TODO: Assume body is always a Ref
- assert isinstance( body.terms.value, ast.Ref), f"assert14 isinstance"
+ assert isinstance( body.terms.value, ast.Ref), "assert14 isinstance"
values = _resolve_ref(
body.terms.value, var_in_body.value.value, opa_request_input
)
@@ -184,9 +184,9 @@ def _resolve_ref(ref, expected_variable_name, opa_request_input):
logging.setLogRecordFactory(correlation.set_correlation_id(context["correlation_id"]))
logger = logging.getLogger(__name__)
logger.debug(f"resolve ref: {ref}")
- assert isinstance(ref.operand(0), ast.Term), f"assert15 isinstance"
- assert isinstance(ref.operand(0).value, ast.Var), f"assert16 isinstance"
- assert ref.operand(0).value.value == "data", f"assert17 data"
+ assert isinstance(ref.operand(0), ast.Term), "assert15 isinstance"
+ assert isinstance(ref.operand(0).value, ast.Var), "assert16 isinstance"
+ assert ref.operand(0).value.value == "data", "assert17 data"
rego_package_name = ref.operand(1).value.value
rego_rule_name = ref.operand(2).value.value
variable_name = ref.operand(3).value.value
@@ -200,7 +200,7 @@ def _resolve_http(expr):
logging.setLogRecordFactory(correlation.set_correlation_id(context["correlation_id"]))
logger = logging.getLogger(__name__)
logger.debug(f"resolve http: {expr}")
- assert isinstance(expr.terms[1].value, ast.Object), f"assert19 isinstance"
+ assert isinstance(expr.terms[1].value, ast.Object), "assert19 isinstance"
method = None
url = None
headers = {}
@@ -220,8 +220,8 @@ def _resolve_http(expr):
pass # no-op
else:
raise NotImplementedError("Unsupported http.send option", str(key))
- assert method is not None, f"assert20 method is None"
- assert url is not None, f"assert21 url is None"
+ assert method is not None, "assert20 method is None"
+ assert url is not None, "assert21 url is None"
logger.debug(f"http request method: {method}, url: {url}, headers: {headers}")
resp = requests.request(method, url=url, headers=headers)
response = {
@@ -294,4 +294,4 @@ def _postprocess_es_queries(es_queries):
return new_es_queries
new_es_queries = combine_term_queries(es_queries)
- return new_es_queries
\ No newline at end of file
+ return new_es_queries
diff --git a/app/views/home.py b/app/views/home.py
index b1ec5c0c9be3ddd8eda2de51dd891c481e2a05fd..a5985d47c1e4d5b015eea8481896713d883d834d 100644
--- a/app/views/home.py
+++ b/app/views/home.py
@@ -2,7 +2,7 @@ import fastapi
import sys
import os
sys.path.append(os.path.abspath('..'))
-from _buildinfo import version
+import _version
router = fastapi.APIRouter()
def generate_html_response():
@@ -12,7 +12,7 @@ def generate_html_response():
<title>Policy Service</title>
</head>
<body>
- <h1>Policy Service v{version}</h1>
+ <h1>Policy Service v{_version.__version__} {_version.__milestone__}</h1>
<ul>
<li><a href='/api/policy/v1/docs'>Swagger UI</a>
<li><a href='/api/policy/v1/redoc'>ReDoc</a>
diff --git a/devops/aws/opa/values.yaml b/devops/aws/opa/values.yaml
index 58b8aede49bc1753d1ad13736bc464802c268da7..eedc1e7436cddb8da33504d372f54dd8aa999c91 100644
--- a/devops/aws/opa/values.yaml
+++ b/devops/aws/opa/values.yaml
@@ -12,4 +12,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-image: openpolicyagent/opa:0.54.0
\ No newline at end of file
+image: openpolicyagent/opa:0.57.0
\ No newline at end of file
diff --git a/devops/aws/override-stages.yaml b/devops/aws/override-stages.yaml
index 9a12a645074bd9b2fbfb57eeb47b07df6e1d2461..8c62a8f32d10860f5eb691d189d9ba08eccf8159 100644
--- a/devops/aws/override-stages.yaml
+++ b/devops/aws/override-stages.yaml
@@ -14,6 +14,9 @@
aws-containerize:
before_script:
+ - apt-get install -y software-properties-common
+ - apt-get install -y python3 python3-pip
+ - python3 -m pip install -r requirements_setversion.txt
- cd app
- make build_docker
- cd ..
diff --git a/devops/azure/override-stages.yml b/devops/azure/override-stages.yml
index 56ca8d1407c368b8717d72ff4b593d6ecff09ce6..42ee7ba4bd737fd06454b5577bb5e127e3656c82 100644
--- a/devops/azure/override-stages.yml
+++ b/devops/azure/override-stages.yml
@@ -15,12 +15,16 @@ azure_containerize:
- docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
- az --version
- az login --service-principal -u $AZURE_PRINCIPAL_ID -p $AZURE_PRINCIPAL_SECRET --tenant $AZURE_TENANT_ID
+ # Set Version
+ - apk add --no-cache python3 py3-pip
+ - pip install -r requirements_setversion.txt
script:
+ - python3 setversion.py app
# Gitlab Container Registry
- docker build -f $AZURE_BUILD_SUBDIR/Dockerfile -t $CI_REGISTRY_IMAGE/$SHA_IMAGE .
- docker push ${CI_REGISTRY_IMAGE}/$SHA_IMAGE
- echo $CI_COMMIT_TAG
- # Azure Container Registryb
+ # Azure Container Registry
- az acr login -n $AZURE_REGISTRY
- docker tag $CI_REGISTRY_IMAGE/$SHA_IMAGE ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
- docker push ${AZURE_REGISTRY}.azurecr.io/$SHA_IMAGE
@@ -103,7 +107,7 @@ azure_test_py:
- echo "DATA_PARTITION $DATA_PARTITION_ID"
- echo "BUNDLE_PAUSE $BUNDLE_PAUSE"
- echo "CLOUD_PROVIDER $CLOUD_PROVIDER"
- - python3 -m pytest --token=$BEARER_TOKEN --service_url=https://${AZURE_DNS_NAME} --data_partition=$DATA_PARTITION_ID
+ - python3 -m pytest -v --color=yes --token=$BEARER_TOKEN --service_url=https://${AZURE_DNS_NAME} --data_partition=$DATA_PARTITION_ID
only:
variables:
- $AZURE == '1'
diff --git a/devops/gc/bootstrap-osdu-module/Dockerfile b/devops/gc/bootstrap-osdu-module/Dockerfile
index ccc831b2994ec930d551dcf585610e614e4bf4c7..5d0d37da46504acf42ea71f00ef99174c5938261 100644
--- a/devops/gc/bootstrap-osdu-module/Dockerfile
+++ b/devops/gc/bootstrap-osdu-module/Dockerfile
@@ -6,7 +6,7 @@ COPY ./deployment/ /opt/deployment
COPY ./devops/gc/bootstrap-osdu-module /opt/devops/gc/bootstrap-osdu-module
RUN chmod 775 /opt/bootstrap_policy.sh
-RUN apk add py3-pip wget
+RUN apk add py3-pip wget jq
RUN pip3 install -r /opt/requirements_bootstrap.txt -r /opt/devops/gc/bootstrap-osdu-module/requirements.txt
RUN wget --quiet https://dl.min.io/client/mc/release/linux-amd64/mc && chmod +x mc && mv mc /usr/bin/mc
diff --git a/devops/gc/bootstrap-osdu-module/bootstrap_policy.sh b/devops/gc/bootstrap-osdu-module/bootstrap_policy.sh
index d8e87f4ce0ef55ecea8ef5b6e8c3c2b87cd0c9a3..3c9f1aa74e238b24766f3ecbb66ad776419f84bf 100644
--- a/devops/gc/bootstrap-osdu-module/bootstrap_policy.sh
+++ b/devops/gc/bootstrap-osdu-module/bootstrap_policy.sh
@@ -5,48 +5,50 @@
set -ex
+source ./validate-env.sh "PARTITION_BASE_URL"
+
create_instance_bundles() {
- # Renders and archives intance level policies
- echo "Archiving bundle of instance policies..."
- tar -czf bundle.tar.gz --directory='/opt/deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose
- mkdir --parents /opt/policies ; mv bundle.tar.gz "$_"
- echo "Instance policies archive is ready"
+ # Renders and archives intance level policies
+ echo "Archiving bundle of instance policies..."
+ tar -czf bundle.tar.gz --directory='/opt/deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose
+ mkdir --parents /opt/policies
+ mv bundle.tar.gz "$_"
+ echo "Instance policies archive is ready"
}
create_partition_bundle() {
- # Renders and archives policies for data_partition
- # Creates archive named bundle-<data_partition>.tar.gz in /opt/policies
- # Args: $1 - data_partition_id
+ # Renders and archives policies for data_partition
+ # Creates archive named bundle-<data_partition>.tar.gz in /opt/policies
+ # Args: $1 - data_partition_id
- DATA_PARTITION=$1
- echo "Archiving bundle of policies for parition: ${DATA_PARTITION}..."
- python3 /opt/devops/gc/bootstrap-osdu-module/DataPartitionBundles.py --partition "${DATA_PARTITION}"
- mv /opt/bundle-"${DATA_PARTITION}".tar.gz /opt/policies
- echo "${DATA_PARTITION} partition archive is ready"
+ DATA_PARTITION=$1
+ echo "Archiving bundle of policies for parition: ${DATA_PARTITION}..."
+ python3 /opt/devops/gc/bootstrap-osdu-module/DataPartitionBundles.py --partition "${DATA_PARTITION}"
+ mv /opt/bundle-"${DATA_PARTITION}".tar.gz /opt/policies
+ echo "${DATA_PARTITION} partition archive is ready"
}
bootstrap_gcs() {
- echo "Push archives to GCS bucket"
- gsutil cp -n /opt/policies/* gs://"${POLICY_BUCKET}"/
- echo "Bootstrap finished successfully"
+ echo "Push archives to GCS bucket"
+ gsutil cp -n /opt/policies/* gs://"${POLICY_BUCKET}"/
+ echo "Bootstrap finished successfully"
}
bootstrap_minio() {
- echo "Configuring mc tool"
- mc alias set minio "${MINIO_HOST}":"${MINIO_PORT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"
- echo "Pushing archives to Minio bucket"
- for file in /opt/policies/*;
- do
- echo "Processing $file:"
- file_name=${file##*/}
- # Check if file already exists
- if mc stat minio/"${POLICY_BUCKET}"/"$file_name" > /dev/null 2>&1; then
- echo "Skipping $file: already exists in bucket"
- else
- mc cp "$file" minio/"${POLICY_BUCKET}"/"$file_name"
- fi
- done
- echo "Bootstrap finished successfully"
+ echo "Configuring mc tool"
+ mc alias set minio "${MINIO_HOST}":"${MINIO_PORT}" "${MINIO_ACCESS_KEY}" "${MINIO_SECRET_KEY}"
+ echo "Pushing archives to Minio bucket"
+ for file in /opt/policies/*; do
+ echo "Processing $file:"
+ file_name=${file##*/}
+ # Check if file already exists
+ if mc stat minio/"${POLICY_BUCKET}"/"$file_name" >/dev/null 2>&1; then
+ echo "Skipping $file: already exists in bucket"
+ else
+ mc cp "$file" minio/"${POLICY_BUCKET}"/"$file_name"
+ fi
+ done
+ echo "Bootstrap finished successfully"
}
# Main part
@@ -54,27 +56,28 @@ bootstrap_minio() {
source ./validate-env.sh "DATA_PARTITION"
source ./validate-env.sh "POLICY_BUCKET"
-## Creating instance bundles
+# Creating instance bundles
create_instance_bundles
-## Creating partition bundles
-IFS=',' read -ra PARTITIONS <<< "${DATA_PARTITION_ID_LIST}"
-PARTITIONS=("${PARTITIONS[@]}")
+# Get all partitions
+PARTITIONS_LIST=$(curl --location "${PARTITION_BASE_URL}/api/partition/v1/partitions" | jq -r '[.[] | select(. != "system")] | join(",")')
+IFS=',' read -ra PARTITIONS <<< "${PARTITIONS_LIST}"
+echo $PARTITIONS
+# Creating partition bundles
for PARTITION in "${PARTITIONS[@]}"; do
- create_partition_bundle "${PARTITION}"
+ create_partition_bundle "${PARTITION}"
done
-## Uploading bundles to gcs/minio bucket
-if [ "${ONPREM_ENABLED}" == "true" ]
-then
- source ./validate-env.sh "MINIO_HOST"
- source ./validate-env.sh "MINIO_ACCESS_KEY"
- source ./validate-env.sh "MINIO_SECRET_KEY"
- source ./validate-env.sh "MINIO_PORT"
- bootstrap_minio
+# Uploading bundles to gcs/minio bucket
+if [ "${ONPREM_ENABLED}" == "true" ]; then
+ source ./validate-env.sh "MINIO_HOST"
+ source ./validate-env.sh "MINIO_ACCESS_KEY"
+ source ./validate-env.sh "MINIO_SECRET_KEY"
+ source ./validate-env.sh "MINIO_PORT"
+ bootstrap_minio
else
- bootstrap_gcs
+ bootstrap_gcs
fi
touch /tmp/bootstrap_ready
diff --git a/devops/gc/deploy/README.md b/devops/gc/deploy/README.md
index bddda2ef08f5c8d671c9953ff8572d79693d33d2..5cef8f877358a827c14a0a238727e7b6452fee59 100644
--- a/devops/gc/deploy/README.md
+++ b/devops/gc/deploy/README.md
@@ -48,17 +48,18 @@ First you need to set variables in **values.yaml** file using any code editor. S
**data.serviceAccountName** | name of your service account | string | - | yes
**data.imagePullPolicy** | when to pull image | string | IfNotPresent | yes
**data.bucketName** | bucket name | string | - | yes
-**data.scopes** | scope of OPA | string | "https://www.googleapis.com/auth/devstorage.read_only" | yes
-**data.entitlementsHost** | Entitlements host | string | "http://entitlements" | yes
+**data.scopes** | scope of OPA | string | "<https://www.googleapis.com/auth/devstorage.read_only>" | yes
+**data.entitlementsHost** | Entitlements host | string | "<http://entitlements>" | yes
**data.entitlementsBasePath** | Entitlements path | string | "/api/entitlements/v2/groups" | yes
**data.useBundles** | use bundle or not | string | "yes" | yes
-**data.legalHost** | Legal host | string | "http://legal" | yes
+**data.legalHost** | Legal host | string | "<http://legal>" | yes
+**data.partitionHost** | Partition host | string | "<http://partition>" | yes
### Baremetal variables
| Name | Description | Type | Default |Required |
|------|-------------|------|---------|---------|
-**data.minioHost** | minio host | string | http://minio:9000 | yes
+**data.minioHost** | minio host | string | <http://minio:9000> | yes
**conf.minioSecretName** | secret name for the app | string | "policy-minio-secret" | yes
### Config variables
@@ -69,7 +70,7 @@ First you need to set variables in **values.yaml** file using any code editor. S
**conf.configmap** | configmap to be used | string | policy-config | yes
**conf.bootstrapSecretName** | secret name for the bootstrap | string | "minio-bootstrap-secret" | yes
**data.dataPartitionId** | data partition id | string | - | yes
-**data.dataPartitionIdList** | list of secondary data partition ids in case of multipartition | string | - | no
+**data.dataPartitionIdList** | list of secondary data partition ids in case of multipartition | string | - | yes
**conf.minDelaySeconds** | min delay for bundle download | num | 6 | yes
**conf.maxDelaySeconds** | max delay for bundle download | num | 12 | yes
diff --git a/devops/gc/deploy/templates/opa-configmap.yaml b/devops/gc/deploy/templates/opa-configmap.yaml
deleted file mode 100644
index 335a9c78a88866817463b32b4c8c18e413976e3c..0000000000000000000000000000000000000000
--- a/devops/gc/deploy/templates/opa-configmap.yaml
+++ /dev/null
@@ -1,48 +0,0 @@
-apiVersion: v1
-kind: ConfigMap
-metadata:
- labels:
- app: "{{ .Values.opa.conf.appName }}"
- name: "{{ .Values.opa.conf.configmap }}"
- namespace: "{{ .Release.Namespace }}"
-data:
-{{- if not .Values.global.onPremEnabled }}
- config.yaml: |
- services:
- gcs:
- url: "https://storage.googleapis.com/storage/v1/b/{{ .Values.data.bucketName }}/o"
- credentials:
- gcp_metadata:
- scopes:
- - "{{ .Values.data.scopes }}"
- bundles:
- osdu/instance:
- service: gcs
- # NOTE ?alt=media is required
- resource: 'bundle.tar.gz?alt=media'
- {{- range (compact .Values.data.dataPartitionIdList) }}
- osdu/partition/{{ . }}:
- service: gcs
- resource: 'bundle-{{ . }}.tar.gz?alt=media'
- polling:
- min_delay_seconds: {{ $.Values.conf.minDelaySeconds }}
- max_delay_seconds: {{ $.Values.conf.maxDelaySeconds }}
- {{- end }}
-{{- else }}
- config.yaml: |
- services:
- s3:
- url: http://minio:9000/{{ .Values.data.bucketName }}
- credentials:
- s3_signing:
- environment_credentials: {}
- bundles:
- osdu/instance:
- service: s3
- resource: bundle.tar.gz
- {{- range (compact .Values.data.dataPartitionIdList) }}
- osdu/partition/{{ . }}:
- service: s3
- resource: 'bundle-{{ . }}.tar.gz'
- {{- end }}
-{{- end }}
diff --git a/devops/gc/deploy/templates/opa-deployment.yaml b/devops/gc/deploy/templates/opa-deployment.yaml
index 44e91b5d4366eae92ac6519e087b6db3d5a63fb7..a83f374d2dd765af9e9186362bed0db0ae49dee3 100644
--- a/devops/gc/deploy/templates/opa-deployment.yaml
+++ b/devops/gc/deploy/templates/opa-deployment.yaml
@@ -6,7 +6,7 @@ metadata:
app: "{{ .Values.opa.conf.appName }}"
namespace: "{{ .Release.Namespace }}"
spec:
- replicas: 1
+ replicas: {{ .Values.opa.conf.replicas }}
strategy:
type: Recreate
selector:
@@ -24,6 +24,17 @@ spec:
sidecar.istio.io/proxyMemoryLimit: {{ .Values.istio.proxyMemoryLimit | quote }}
name: "{{ .Values.opa.conf.appName }}"
spec:
+ initContainers:
+ - name: "{{ .Values.opa.init.name }}"
+ image: "{{ .Values.opa.init.image }}"
+ securityContext:
+ runAsUser: 1337
+ volumeMounts:
+ - name: config
+ mountPath: /config
+ envFrom:
+ - configMapRef:
+ name: {{ printf "%s-bootstrap" .Values.conf.configmap | quote }}
containers:
- name: "{{ .Values.opa.conf.appName }}"
image: "{{ .Values.opa.data.image }}"
@@ -45,8 +56,8 @@ spec:
memory: "{{ .Values.data.limitsMemory }}"
{{- end }}
volumeMounts:
- - mountPath: /config
- name: "{{ .Values.opa.conf.configmap }}"
+ - name: config
+ mountPath: /config
envFrom:
- configMapRef:
name: "{{ .Values.opa.conf.envConfig }}"
@@ -72,7 +83,6 @@ spec:
allowPrivilegeEscalation: false
runAsNonRoot: true
volumes:
- - name: "{{ .Values.opa.conf.configmap }}"
- configMap:
- name: "{{ .Values.opa.conf.configmap }}"
+ - name: config
+ emptyDir: {}
serviceAccountName: "{{ .Values.opa.data.serviceAccountName }}"
diff --git a/devops/gc/deploy/templates/policy-configmap-bootstrap.yaml b/devops/gc/deploy/templates/policy-configmap-bootstrap.yaml
index cacbcd3e7e6ba23b89585dbcddee8e911c6a849b..ff6419c10aa961b382af9e7e0b45244b83bdbd39 100644
--- a/devops/gc/deploy/templates/policy-configmap-bootstrap.yaml
+++ b/devops/gc/deploy/templates/policy-configmap-bootstrap.yaml
@@ -8,5 +8,5 @@ metadata:
data:
POLICY_BUCKET: "{{ .Values.data.bucketName }}"
DATA_PARTITION: "{{ .Values.data.dataPartitionId }}"
- DATA_PARTITION_ID_LIST: {{ join "," .Values.data.dataPartitionIdList | quote }}
+ PARTITION_BASE_URL: {{ .Values.data.partitionHost | quote }}
ONPREM_ENABLED: "{{ .Values.global.onPremEnabled }}"
diff --git a/devops/gc/deploy/templates/policy-deployment.yaml b/devops/gc/deploy/templates/policy-deployment.yaml
index a2d88fbe4edaedbc7da7cc1ba01cc3612da380f3..a7df1a3c027ece71a95d4b335e049d95f35dc70b 100644
--- a/devops/gc/deploy/templates/policy-deployment.yaml
+++ b/devops/gc/deploy/templates/policy-deployment.yaml
@@ -9,7 +9,7 @@ spec:
selector:
matchLabels:
app: "{{ .Values.conf.appName }}"
- replicas: 2
+ replicas: {{ .Values.conf.replicas }}
template:
metadata:
labels:
@@ -37,6 +37,13 @@ spec:
runAsNonRoot: true
ports:
- containerPort: 8080
+ livenessProbe:
+ failureThreshold: 3
+ httpGet:
+ path: api/policy/v1/health
+ port: 8080
+ initialDelaySeconds: 120
+ periodSeconds: 10
resources:
requests:
cpu: "{{ .Values.data.requestsCpu }}"
diff --git a/devops/gc/deploy/values.yaml b/devops/gc/deploy/values.yaml
index d25a31082d736a44b5ec61f4e6f57701f992726d..f7e9eb395deddf780083376d4ca10a7546eff7b0 100644
--- a/devops/gc/deploy/values.yaml
+++ b/devops/gc/deploy/values.yaml
@@ -1,55 +1,53 @@
# Common values for all deployments
global:
- domain: ""
+ domain: ''
onPremEnabled: false
limitsEnabled: true
-
data:
# Deployment resources
- requestsCpu: "5m"
- requestsMemory: "1.2G"
- limitsCpu: "1"
- limitsMemory: "1.5G"
- serviceAccountName: "policy"
- imagePullPolicy: "IfNotPresent"
- image: ""
- bootstrapImage: ""
- bootstrapServiceAccountName: ""
+ requestsCpu: 5m
+ requestsMemory: 1.2G
+ limitsCpu: '1'
+ limitsMemory: 1.5G
+ serviceAccountName: policy
+ imagePullPolicy: IfNotPresent
+ image: ''
+ bootstrapImage: ''
+ bootstrapServiceAccountName: ''
# ConfigMap resources
- logLevel: "ERROR"
- entitlementsHost: "http://entitlements"
- entitlementsBasePath: "/api/entitlements/v2/groups"
- legalHost: "http://legal"
- bucketName: ""
- useBundles: "yes"
- dataPartitionId: ""
- dataPartitionIdList: ["osdu"]
- scopes: "https://www.googleapis.com/auth/devstorage.read_only"
+ logLevel: ERROR
+ entitlementsHost: http://entitlements
+ entitlementsBasePath: /api/entitlements/v2/groups
+ legalHost: http://legal
+ partitionHost: http://partition
+ bucketName: ''
+ useBundles: 'yes'
+ dataPartitionId: ''
# baremetal only
- minioHost: "http://minio:9000"
-
+ minioHost: http://minio:9000
conf:
- appName: "policy"
- configmap: "policy-config"
- minioSecretName: "policy-minio-secret"
- bootstrapSecretName: "minio-bootstrap-secret"
- minDelaySeconds: 6
- maxDelaySeconds: 12
-
+ appName: policy
+ configmap: policy-config
+ minioSecretName: policy-minio-secret
+ bootstrapSecretName: minio-bootstrap-secret
+ replicas: 2
opa:
data:
- requestsMemory: "200Mi"
- image: "docker.io/openpolicyagent/opa:latest-rootless"
- serviceAccountName: "opa"
+ requestsMemory: 200Mi
+ image: docker.io/openpolicyagent/opa:latest-rootless
+ serviceAccountName: opa
conf:
- configmap: "opa-config"
- envConfig: "opa-env-config"
- appName: "opa"
+ envConfig: opa-env-config
+ appName: opa
+ replicas: 1
+ init:
+ name: init-config
+ image: ''
istio:
- proxyCPU: "10m"
- proxyCPULimit: "500m"
- proxyMemory: "50Mi"
- proxyMemoryLimit: "512Mi"
- bootstrapProxyCPU: "5m"
- bootstrapProxyCPULimit: "100m"
+ proxyCPU: 10m
+ proxyCPULimit: 500m
+ proxyMemory: 50Mi
+ proxyMemoryLimit: 512Mi
+ bootstrapProxyCPU: 5m
+ bootstrapProxyCPULimit: 100m
diff --git a/devops/gc/init-container-opa/Dockerfile b/devops/gc/init-container-opa/Dockerfile
new file mode 100644
index 0000000000000000000000000000000000000000..8fac6b566631b607d27439aaf29f3d6de9bea28e
--- /dev/null
+++ b/devops/gc/init-container-opa/Dockerfile
@@ -0,0 +1,18 @@
+FROM alpine:latest
+
+WORKDIR /opt
+
+COPY ./devops/gc/init-container-opa/ /opt/
+
+RUN chmod 775 init_opa.sh validate-env.sh
+
+RUN apk add jq bash curl
+
+RUN addgroup -g 10001 -S nonroot \
+ && adduser -h /opt -G nonroot -S -u 10001 nonroot
+
+RUN chown -R 10001:10001 /opt
+
+USER 10001:10001
+
+CMD ["/bin/bash", "-c", "/opt/init_opa.sh"]
diff --git a/devops/gc/init-container-opa/init_opa.sh b/devops/gc/init-container-opa/init_opa.sh
new file mode 100644
index 0000000000000000000000000000000000000000..8c1f3b1a0ec54cce45a509a3452b56be139e4f62
--- /dev/null
+++ b/devops/gc/init-container-opa/init_opa.sh
@@ -0,0 +1,82 @@
+#!/usr/bin/env bash
+
+source ./validate-env.sh "PARTITION_BASE_URL"
+source ./validate-env.sh "POLICY_BUCKET"
+source ./validate-env.sh "ONPREM_ENABLED"
+
+# Get all partitions
+status_code=$(curl -X GET \
+ --url "${PARTITION_BASE_URL}/api/partition/v1/partitions" --write-out "%{http_code}" --silent --output "/dev/null" \
+ -H "Content-Type: application/json")
+
+if [[ "${status_code}" == 200 ]]; then
+ PARTITIONS_LIST=$(curl --location "${PARTITION_BASE_URL}/api/partition/v1/partitions" | jq -r '[.[] | select(. != "system")] | join(",")')
+ IFS=',' read -ra PARTITIONS <<< "${PARTITIONS_LIST}"
+ echo $PARTITIONS
+ else
+ echo "Exiting with status code: ${status_code}"
+ sleep 15
+ exit 1
+ fi
+
+# Function for create config.yaml for OPA
+create_config_gc() {
+echo "Creating config file for OPA..."
+cat << EOF > /config/config.yaml
+services:
+ gcs:
+ url: "https://storage.googleapis.com/storage/v1/b/$POLICY_BUCKET/o"
+ credentials:
+ gcp_metadata:
+ scopes:
+ - "https://www.googleapis.com/auth/devstorage.read_only"
+bundles:
+ osdu/instance:
+ service: gcs
+ # NOTE ?alt=media is required
+ resource: 'bundle.tar.gz?alt=media'
+EOF
+
+for PARTITION in "${PARTITIONS[@]}"; do
+cat << EOF >> /config/config.yaml
+ osdu/partition/$PARTITION:
+ service: gcs
+ resource: 'bundle-$PARTITION.tar.gz?alt=media'
+ polling:
+ min_delay_seconds: 6
+ max_delay_seconds: 12
+EOF
+done
+echo "Config for OPA is ready"
+}
+
+create_config_onprem() {
+echo "Creating config file for OPA..."
+cat << EOF > /config/config.yaml
+services:
+ s3:
+ url: http://minio:9000/$POLICY_BUCKET
+ credentials:
+ s3_signing:
+ environment_credentials: {}
+bundles:
+ osdu/instance:
+ service: s3
+ resource: bundle.tar.gz
+EOF
+
+for PARTITION in "${PARTITIONS[@]}"; do
+cat << EOF >> /config/config.yaml
+ osdu/partition/$PARTITION:
+ service: s3
+ resource: 'bundle-$PARTITION.tar.gz'
+EOF
+done
+echo "Config for OPA is ready"
+}
+
+if [ "${ONPREM_ENABLED}" == "true" ]; then
+ create_config_onprem
+else
+ create_config_gc
+fi
diff --git a/devops/gc/init-container-opa/validate-env.sh b/devops/gc/init-container-opa/validate-env.sh
new file mode 100644
index 0000000000000000000000000000000000000000..9de17ff4a83b4e297bba6aec875088dd313c416b
--- /dev/null
+++ b/devops/gc/init-container-opa/validate-env.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+{ set +x ;} 2> /dev/null # disable output to prevent secret logging
+set -e
+
+ENV_VAR_NAME=$1
+
+if [ "${!ENV_VAR_NAME}" = "" ]
+then
+ echo "Missing environment variable '$ENV_VAR_NAME'. Please provide all variables and try again"
+ { set -x ;} 2> /dev/null # enable output back
+ exit 1
+fi
+
+{ set -x ;} 2> /dev/null # enable output back
diff --git a/devops/gc/pipeline/override-stages.yml b/devops/gc/pipeline/override-stages.yml
index 32f4d987770684db76a2368aca98b48e26bb536f..36523c44c4114601d820e0750863307b644e09b0 100644
--- a/devops/gc/pipeline/override-stages.yml
+++ b/devops/gc/pipeline/override-stages.yml
@@ -7,11 +7,70 @@ variables:
GC_INT_TEST_TYPE: python
GC_PYTHON_INT_TEST_SUBDIR: "app/tests/gc"
GC_BAREMETAL_PYTHON_INT_TEST_SUBDIR: "app/tests/baremetal"
+ GC_INIT_IMAGE_NAME: gc-opa-init-container
# Workaround since tests by default extends .maven. Need it for tests when disabling other CSPs
.maven:
variables:
+.gc_set_image_name:
+ script:
+ - >
+ if echo $CI_COMMIT_REF_NAME | grep -Eq "^release\/[0-9]{1,2}.[0-9]{1,2}$";
+ then
+ export IMAGE_NAME=$IMAGE_NAME-release;
+ export IMAGE_BOOTSTRAP_NAME=$IMAGE_BOOTSTRAP_NAME-release;
+ export GC_INIT_IMAGE_NAME=$GC_INIT_IMAGE_NAME-release;
+ fi
+ - >
+ if [[ "$CI_COMMIT_REF_NAME" == "$CI_DEFAULT_BRANCH" ]];
+ then
+ export IMAGE_NAME=$IMAGE_NAME-master;
+ export IMAGE_BOOTSTRAP_NAME=$IMAGE_BOOTSTRAP_NAME-master;
+ export GC_INIT_IMAGE_NAME=$GC_INIT_IMAGE_NAME-master;
+ fi
+ - >
+ if [[ "$CI_COMMIT_TAG" != "" ]];
+ then
+ IMAGE_TAG="$CI_COMMIT_TAG";
+ EXTRA_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_TAG";
+ EXTRA_BOOTSTRAP_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME:$CI_COMMIT_TAG";
+ EXTRA_INIT_TAG="-t $CI_REGISTRY_IMAGE/$GC_INIT_IMAGE_NAME:$CI_COMMIT_TAG";
+ elif [[ "$CI_COMMIT_REF_NAME" == "$CI_DEFAULT_BRANCH" ]];
+ then
+ IMAGE_TAG="$CI_COMMIT_SHORT_SHA";
+ HELM_TAG="latest"
+ EXTRA_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA -t $CI_REGISTRY_IMAGE/$IMAGE_NAME:latest";
+ EXTRA_BOOTSTRAP_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME:$CI_COMMIT_SHORT_SHA -t $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME:latest";
+ EXTRA_INIT_TAG="-t $CI_REGISTRY_IMAGE/$GC_INIT_IMAGE_NAME:$CI_COMMIT_SHORT_SHA -t $CI_REGISTRY_IMAGE/$GC_INIT_IMAGE_NAME:latest";
+ else
+ IMAGE_TAG="$CI_COMMIT_SHORT_SHA";
+ HELM_TAG="gc$CI_COMMIT_SHORT_SHA"
+ EXTRA_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_NAME:$CI_COMMIT_SHORT_SHA";
+ EXTRA_BOOTSTRAP_TAG="-t $CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME:$CI_COMMIT_SHORT_SHA";
+ EXTRA_INIT_TAG="-t $CI_REGISTRY_IMAGE/$GC_INIT_IMAGE_NAME:$CI_COMMIT_SHORT_SHA";
+ fi
+
+.gc_substitute_image_in_helm:
+ script:
+ - wget -q https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -O /usr/bin/yq && chmod +x /usr/bin/yq
+ - IMAGE="$CI_REGISTRY_IMAGE/$IMAGE_NAME:$IMAGE_TAG" yq -i '.data.image = env(IMAGE)' $GC_HELM_DEPLOYMENT_DIR/values.yaml
+ - yq -e '.data | has("bootstrapImage")' $GC_HELM_DEPLOYMENT_DIR/values.yaml &>/dev/null && BOOTSTRAP_IMAGE="$CI_REGISTRY_IMAGE/$IMAGE_BOOTSTRAP_NAME:$IMAGE_TAG" yq -i '.data.bootstrapImage = env(BOOTSTRAP_IMAGE)' $GC_HELM_DEPLOYMENT_DIR/values.yaml
+ - INIT_IMAGE="$CI_REGISTRY_IMAGE/$GC_INIT_IMAGE_NAME:$IMAGE_TAG" yq -i '.opa.init.image = env(INIT_IMAGE)' $GC_HELM_DEPLOYMENT_DIR/values.yaml
+ - cat $GC_HELM_DEPLOYMENT_DIR/values.yaml | grep -i image
+
+gc-containerize-init-gitlab:
+ extends: gc-containerize-gitlab
+ variables:
+ IMAGE_NAME: "$GC_INIT_IMAGE_NAME"
+ BUILD_PATH: "devops/gc/init-container-opa/Dockerfile"
+
+gc-containerize-init-gcr:
+ extends: gc-containerize-gcr
+ variables:
+ IMAGE_NAME: "$GC_INIT_IMAGE_NAME"
+ BUILD_PATH: "devops/gc/init-container-opa/Dockerfile"
+
gc-test-python:
variables:
POLICY_BUCKET: $GC_POLICY_BUCKET
diff --git a/docs/docs/bundles.md b/docs/docs/bundles.md
index d5488b8b30899291f1b3d4b65436ec7d22f4994f..e414670fb9929575279d6d379c93a6c6e5b09d48 100644
--- a/docs/docs/bundles.md
+++ b/docs/docs/bundles.md
@@ -3,7 +3,7 @@
!!! warning
With bundles enabled (the default), it is important that you validate the rego contained in the bootstrap file and when updating policies via policy service APIs.
-If there are syntax errors in the bundle, OPA will refuse to load them into memory which could break the policy service.
+If there are syntax errors in the bundle, OPA will refuse to load them into memory which could break the policy service. In M21 rego validation was introduced for PUT API requests, which is on by default but can be optionally disabled (see `ENABLE_VERIFY_POLICY`` for more information).
## Types of bundles and bundle bootstrapping
@@ -53,6 +53,12 @@ records := data.osdu.instance.dataauthz.records
```
Examples of the above can be found in [tests template directory](https://community.opengroup.org/osdu/platform/security-and-compliance/policy/-/tree/master/app/tests/templates).
+## Rate limiting
+
+!!! warning
+
+ Policy Service does not implement a retry mechanism when writing to the bundle service. In environments where you expect a lot of policy additions, updates and deletes (i.e. test environments) special care should be taken when using rate limits to your bundle service backend as these put/deletes will fail, which could cause your testing to fail with "Error when talking to bundle service" in detail of API response along with a HTTP_503_SERVICE_UNAVAILABLE status code.
+
## Adding a new partition to OSDU
!!! warning
diff --git a/docs/docs/releasenotes.md b/docs/docs/releasenotes.md
index 76b0563dc311d7471687660172ae6dc513204a66..82d58cd2155518173ba81a7adea9cdec910a9f05 100644
--- a/docs/docs/releasenotes.md
+++ b/docs/docs/releasenotes.md
@@ -3,11 +3,12 @@
Policy Service v0.24.0 2023/10
### M21 Breaking Changes
-- Put /policies/osdu/partition/{data_partition}/{policy_id} will now return 202 for accepted updates to policies, that is updated and new policies. If no change is detected it will return a 200. Additionally the return json has changed slightly (message updates, and new SHA1).
+- Put /policies/osdu/partition/{data_partition}/{policy_id} will now return 202 for accepted updates to policies, that is updated and new policies. If no change is detected it will return a 200. Additionally the return json has changed slightly (message updates, new validated and new SHA1 in response).
### M21 Bug Fixes
- X-Debug-Result response header bugfix for use with ENABLE_DEV_DIAGNOSTICS
- Policy Service should handle entitlement service issues gracefully [112](https://community.opengroup.org/osdu/platform/security-and-compliance/policy/-/issues/112)
+- Bugfix for LOCAL mode (developer use) on /config API
### M21 Minor Changes
- Log message updates
diff --git a/loadtest/run-load-tests.sh b/loadtest/run-load-tests.sh
index d65c02f9f605a187546d6f0b3a17d13daf28160c..29bf278670a76c34eafb8feafc9a8cb5fce6f7d9 100755
--- a/loadtest/run-load-tests.sh
+++ b/loadtest/run-load-tests.sh
@@ -29,6 +29,7 @@ if [ ${CLOUD_PROVIDER} == "aws" ]; then
sleep 165
elif [ ${CLOUD_PROVIDER} == "azure" ]; then
echo "## AZURE_DATA_PARTITION = $AZURE_DATA_PARTITION"
+ yum -y install curl jq
python3 -m pip install -r requirements.txt
export TOKEN=$(curl -ks -XPOST "https://login.microsoftonline.com/${AZURE_TENANT_ID}/oauth2/token" -d "grant_type=client_credentials&client_id=${AZURE_PRINCIPAL_ID}&client_secret=${AZURE_PRINCIPAL_SECRET}&resource=${AZURE_APP_ID}" | jq --raw-output '.access_token')
export SERVICE_URL=https://${AZURE_DNS_NAME}
diff --git a/requirements.txt b/requirements.txt
index 8022796dc8ebbaa0ff88291d52cbf4922ad47508..3f96d2ff240295c7702d542b39617838b829dab7 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -18,4 +18,4 @@ google-cloud-storage
# osdu dependences
--extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple
-osdu-api[all]~=0.22.0rc1, ==0.22.* # it will install a rc-version if there is no release one.
+osdu-api[all]~=0.24.0rc1, ==0.24.* # it will install a rc-version if there is no release one.
diff --git a/requirements_dev.txt b/requirements_dev.txt
index 548cdb22245592edff679ee79fe01eb51bd6a5c5..f66bcc6ba2758fa31d3a996d5f52479ff26ca34e 100644
--- a/requirements_dev.txt
+++ b/requirements_dev.txt
@@ -4,6 +4,7 @@ pytest-cov
pytest-mock
httpx
parameterized
+typer
# the following are used in functional integration tests
boto3==1.26.133
@@ -16,4 +17,4 @@ msal
# osdu dependences
--extra-index-url https://community.opengroup.org/api/v4/projects/148/packages/pypi/simple
-osdu-api[all]~=0.22.0rc1, ==0.22.*
+osdu-api[all]~=0.24.0rc1, ==0.24.*
diff --git a/requirements_setversion.txt b/requirements_setversion.txt
new file mode 100644
index 0000000000000000000000000000000000000000..ec184f68004fb9227b937d7416819dac5eba2765
--- /dev/null
+++ b/requirements_setversion.txt
@@ -0,0 +1 @@
+typer
diff --git a/setversion.py b/setversion.py
new file mode 100755
index 0000000000000000000000000000000000000000..9a893d0c14bdd92b4383ac0bbab260fd5c0bdd1c
--- /dev/null
+++ b/setversion.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+import typer
+import time
+from datetime import datetime
+import re
+
+def main(
+ path: str = typer.Argument(envvar="CI_PROJECT_NAME"),
+ branch: str = typer.Option("dev", "--branch", help="Release", envvar="CI_COMMIT_REF_NAME"),
+ build: str = typer.Option("0", "--build", help="Build ID", envvar="CI_PIPELINE_IID"),
+ commit_message: str = typer.Option("", envvar="CI_COMMIT_MESSAGE"),
+ commit_timestamp: str = typer.Option(None, envvar="CI_COMMIT_TIMESTAMP"),
+ commit_ref_slug: str = typer.Option("", envvar="CI_COMMIT_REF_SLUG}"),
+ commit_id: str = typer.Option("", envvar="CI_COMMIT_SHA"),
+ versionfile: str = typer.Option("VERSION", "--version-file", help="Version file"),
+ pyfile: str = typer.Option("_version.py", "--pyfile", help="python file"),
+ ):
+
+ with open(versionfile, "r") as f:
+ (major, minor, patch) = f.read().strip().split('.')
+
+ if branch == "master" or branch == "main":
+ release = f"rc{patch}.dev{build}"
+ elif branch.startswith("release"):
+ release = f"{patch}"
+ elif branch.startswith("trusted"):
+ release = f"b{patch}.dev{build}"
+ else:
+ release = f"a{patch}.dev{build}"
+
+ __version__ = f"{major}.{minor}.{release}"
+ print(f"{__version__}")
+ milestone = int(minor) - 3
+
+ if not commit_timestamp:
+ today = datetime.now()
+ commit_timestamp = today.isoformat()
+
+ regex = re.compile('[^a-zA-Z0-9_]')
+ commit_message = regex.sub('', commit_message[0:64])
+
+ with open(path + "/" + pyfile, "w") as f:
+ f.write(f"__version__ = \"{__version__}\"\n")
+ f.write(f"__milestone__ = \"M{milestone}\"\n")
+ f.write(f"__branch__ = \"{branch}\"\n")
+ f.write(f"__buildtime__ = {time.time()}\n")
+ f.write(f"__commitid__ = \"{commit_id}\"\n")
+ #f.write(f"__commitmessage__ = \"{commit_message}\"\n")
+ f.write(f"__commitmessage__ = \"\"\n")
+ f.write(f"__committimestamp__ = \"{commit_timestamp}\"\n")
+ f.write(f"__commitrefslug__ = \"{commit_ref_slug}\"\n")
+
+if __name__ == "__main__":
+ typer.run(main)