diff --git a/app/Makefile b/app/Makefile index dc6096cf8b7bc60c86ec04ca65679e465d59a287..d1c1b8592af02c8d3669db815191a74e31d2273f 100644 --- a/app/Makefile +++ b/app/Makefile @@ -174,7 +174,7 @@ gcp_local: POLICY_BUCKET := $(POLICY_BUCKET) gcp_local: ENABLE_DEV_DIAGNOSTICS := 1 gcp_local: ENABLE_ADMIN_UI := 1 gcp_local: gcp_set_token echoenv - - uvicorn main:app --port $(PORT) --reload + uvicorn main:app --port $(PORT) --reload aws_local_blue: OPA_URL := http://localhost:$(OPA_PORT) aws_local_blue: CLOUD_PROVIDER := aws @@ -184,6 +184,21 @@ aws_local_blue: LOG_LEVEL := DEBUG aws_local_blue: aws_set_token_blue echoenv - source $(AWSENV_BLUE) && uvicorn main:app --port $(PORT) --reload +# this is for use when you have backend access to AWS +# be sure to set: +# POLICY_BUCKET +# ENTITLEMENTS_BASE_URL +# LEGAL_BASE_URL +# PARTITION_BASE_URL +# DOMAIN +aws_local: OPA_URL := http://localhost:$(OPA_PORT) +aws_local: CLOUD_PROVIDER := aws +aws_local: ENABLE_DEV_DIAGNOSTICS := 1 +aws_local: ENABLE_ADMIN_UI := 1 +aws_local: LOG_LEVEL := DEBUG +aws_local: echoenv + uvicorn main:app --port $(PORT) --reload + aws_local_green: OPA_URL := http://localhost:$(OPA_PORT) aws_local_green: CLOUD_PROVIDER := aws aws_local_green: ENABLE_DEV_DIAGNOSTICS := 1 diff --git a/app/_buildinfo.py b/app/_buildinfo.py index 972c5ef9da29c640dbe7d2c8cca84f017686eae7..d7c10a5e48eab0224b1b4c16fa2c1c7d56589fef 100644 --- a/app/_buildinfo.py +++ b/app/_buildinfo.py @@ -1,5 +1,6 @@ # IN THE FUTURE THIS WILL BE GENERATED BY CI version = "0.1.4" +milestone = "M18" artifactId = None name = "policy" groupId = "org.opengroup.osdu" diff --git a/app/api/backup_api.py b/app/api/backup_api.py index 2ba17da783121947bb9915575e2c3d8498c721f0..faff9843e2e93f11aeeb39c923d89c067def8251 100644 --- a/app/api/backup_api.py +++ b/app/api/backup_api.py @@ -1,29 +1,43 @@ import fastapi -from fastapi import Depends +from fastapi import Depends, HTTPException import requests from auth import auth import logging import conf from starlette_context import context -from fastapi.responses import FileResponse, Response +from fastapi.responses import StreamingResponse from bundles import bundle router = fastapi.APIRouter() import correlation from datetime import date +from starlette.status import ( + HTTP_405_METHOD_NOT_ALLOWED, + HTTP_424_FAILED_DEPENDENCY +) + _logger = logging.getLogger(__name__) logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": "None"}) -@router.get("/backup", response_class=FileResponse) +@router.get("/backup") def backup(auth_data: auth.Auth = Depends(auth.require_authorized_admin)): """ Experimental Backup API. + + Allows downloading the bundle for a data partition. + + Bundle filename will be in the form bundle-`data partition`-`date`.tar.gz """ logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": context["correlation_id"]}) today = date.today() filename = f"bundle-{auth_data.data_partition_id}-{today}.tar.gz" - file_path = f"bundle-{auth_data.data_partition_id}-{today}.tar.gz" data = bundle.get_bundle(data_partition=auth_data.data_partition_id) - print(data) - return Response(data) - #return FileResponse(path=file_path, filename=filename, media_type='application/gzip') \ No newline at end of file + headers = { + 'Content-Disposition': f"attachment; filename={filename}" + } + if data: + return StreamingResponse(data, headers=headers) + else: + detail = f"Backup of bundle for partition '{auth_data.data_partition_id}' failed!" + logger.error(detail) + raise HTTPException(status_code=HTTP_424_FAILED_DEPENDENCY, detail=detail) \ No newline at end of file diff --git a/app/api/bootstrap_api.py b/app/api/bootstrap_api.py new file mode 100644 index 0000000000000000000000000000000000000000..dc5864d4f0488a215cf3c2e4b0ac99d1dda39b1a --- /dev/null +++ b/app/api/bootstrap_api.py @@ -0,0 +1,118 @@ +import fastapi +from fastapi import Depends, HTTPException, Response +from auth import auth +import logging +import conf +import json +from starlette_context import context +from bundles import bundle +router = fastapi.APIRouter() +import correlation + +from starlette.status import ( + HTTP_201_CREATED, + HTTP_202_ACCEPTED, + HTTP_405_METHOD_NOT_ALLOWED, + HTTP_424_FAILED_DEPENDENCY +) + +_logger = logging.getLogger(__name__) +logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": "None"}) + +@router.post("/bootstrap") +def bootstrap(response: Response, + auth_data: auth.Auth = Depends(auth.require_authorized_admin), + force: bool = False + ): + """ + Experimental bootstrap API for creating and updating bundle to default. + This should be used when adding a partition to OSDU. + + Without force: + + * This method is only allowed if the partition doesn't already have a bundle. + * If the bundle already exists it will return 405 METHOD_NOT_ALLOWED. + * Policy Service can be configured to ignore force. + + May return: + + * HTTP_202_ACCEPTED - updated + * HTTP_201_CREATED - created + * HTTP_405_METHOD_NOT_ALLOWED - not allowed + * HTTP_424_FAILED_DEPENDENCY - bundle server caused failure + """ + logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": context["correlation_id"]}) + if bundle.get_bundle(data_partition=auth_data.data_partition_id): + logger.debug(f"bootstrap bundle already exists") + if force: + if not conf.ALLOW_FORCE_BOOTSTRAP: + raise HTTPException(status_code=HTTP_405_METHOD_NOT_ALLOWED, detail=f"'{auth_data.data_partition_id}' already bootstrapped and forced updates not allowed.") + status = bootstrap_bundle(data_partition=auth_data.data_partition_id) + if status: + response.status_code = HTTP_202_ACCEPTED + detail = f"bootstrap bundle for partition '{auth_data.data_partition_id}' updated to default." + logger.info(detail) + return {"detail": detail} + else: + raise HTTPException(status_code=HTTP_424_FAILED_DEPENDENCY, detail=f"'{auth_data.data_partition_id}' update failed {status}!") + else: + raise HTTPException(status_code=HTTP_405_METHOD_NOT_ALLOWED, detail=f"'{auth_data.data_partition_id}' already bootstrapped.") + else: + status = bootstrap_bundle(data_partition=auth_data.data_partition_id) + if status: + response.status_code = HTTP_201_CREATED + detail = f"bootstrap bundle for partition '{auth_data.data_partition_id}' created." + logger.info(detail) + return {"detail": detail} + else: + detail = f"bootstrap bundle for partition '{auth_data.data_partition_id}' create failed {status}!" + logger.error(detail) + raise HTTPException(status_code=HTTP_424_FAILED_DEPENDENCY, detail=detail) + +def bootstrap_bundle(data_partition): + + # default dataauthz rego for a partition + dataauthz = """\ +package osdu.partition["{}"].dataauthz + +import data.osdu.instance.dataauthz as centralauthz + +records := centralauthz.records""".format(data_partition) + + # default search rego for a partition + search = """\ +package osdu.partition["{}"].search + +default allow = false + +allow = true {{ + input.operation == "view" + # At least one user group needs to be in acl viewers + input.record.acl.viewers[_]==input.groups[_] +}} + +allow = true {{ + input.operation == ["view", "create", "update", "delete", "purge"][_] + # At least one user group needs to be in acl owners + input.record.acl.owners[_]==input.groups[_] +}}""".format(data_partition) + + + # manifest is json + manifest = { + "roots": [f"osdu/partition/{data_partition}"] + } + +# manifest = """\ +#{{ +# "roots": ["osdu/partition/{}"] +#}}""".format(data_partition) + + default_policy_list = { + #".manifest": manifest.encode('ascii'), + ".manifest": json.dumps(manifest, indent=0).encode('ascii'), + "dataauthz.rego": dataauthz.encode('ascii'), + "search.rego": search.encode('ascii') + } + + return bundle.put_default_bundle(data_partition=data_partition, policy_list=default_policy_list) diff --git a/app/bundles/bundle.py b/app/bundles/bundle.py index 05094abd7d4fd421d2f74365dd72022e68bf0fee..3b4ceda73eea00b11d7dd5905d32d1453be6dbef 100644 --- a/app/bundles/bundle.py +++ b/app/bundles/bundle.py @@ -2,6 +2,7 @@ import logging from os import environ import tarfile import io +import time from fastapi import HTTPException from starlette.status import ( @@ -25,8 +26,8 @@ from osdu_api.providers.types import FileLikeObject from opa_response import OpaResponse import correlation -_logger = logging.getLogger(__name__) -logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": "None"}) +logger = logging.getLogger(__name__) +#logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": "None"}) def delete_policy(data_partition, policy_id): """ @@ -80,10 +81,15 @@ def get_bundle(data_partition): Get bundle (for backup) """ bundle = bundle_filename(data_partition) - logger.debug(f"bundle: {bundle}") + logger.debug(f"attempting to get bundle: {bundle}") try: + inbuffer = io.BytesIO() inbuffer, uri = storage.get_storage().download_file(bundle, inbuffer) + inbuffer.seek(0) + return inbuffer + #else: + # logger.critical(f"Error Downloading {bundle} from CSP Storage") except Exception as err: logger.critical(f"Error Downloading {bundle} from CSP Storage: {err}") status_code = HTTP_503_SERVICE_UNAVAILABLE @@ -91,8 +97,39 @@ def get_bundle(data_partition): #template = "An exception of type {0} occurred. Arguments:\n{1!r}" #message = template.format(type(err).__name__, err.args) logger.critical(message) - return None - return inbuffer + return None + +def put_default_bundle(data_partition, policy_list): + """ + Get bundle (for backup) + """ + bundle = bundle_filename(data_partition) + logger.debug(f"attempting to get bundle: {bundle}") + + try: + with io.BytesIO() as outbuffer: + with tarfile.open(mode="w:gz", fileobj=outbuffer) as tarout: + for key in policy_list: + info = tarfile.TarInfo(key) + info.size = len(policy_list[key]) + info.mtime = int(time.time()) + tarout.addfile(info, io.BytesIO(policy_list[key])) + #tarout.addfile(info, io.BytesIO(policy_list[key])) + tarout.close() + outbuffer.seek(0) + if storage.get_storage().upload_file(bundle, outbuffer): + logger.info(f"Bundle {bundle} added to bundle server") + return HTTP_202_ACCEPTED + else: + logger.critical(f"Error uploading {bundle} to CSP Storage") + except Exception as err: + logger.critical(f"Error uploading {bundle} from CSP Storage: {err}") + status_code = HTTP_503_SERVICE_UNAVAILABLE + message = f"Unable to upload bundle {bundle} to bundle server error: {err}" + #template = "An exception of type {0} occurred. Arguments:\n{1!r}" + #message = template.format(type(err).__name__, err.args) + logger.critical(message) + return None def put_policy(data_partition, policy_id, policy): """ diff --git a/app/bundles/providers/aws/storage.py b/app/bundles/providers/aws/storage.py index 7dc6bfad50ec598438fc29529a2ba5b3d99447e7..6ce15613989d25d9ce346f61c4cd2ad6fefc10f3 100644 --- a/app/bundles/providers/aws/storage.py +++ b/app/bundles/providers/aws/storage.py @@ -23,8 +23,8 @@ from bundles.storage import BundleStorageClient import correlation import conf -_logger = logging.getLogger(__name__) -logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": "None"}) +logger = logging.getLogger(__name__) +#logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": "None"}) logger.setLevel(conf.LOG_LEVEL) class FileNotFound(Exception): @@ -39,6 +39,7 @@ class AWSBundleStorageClient(BundleStorageClient): self.content_type = "application/x-gtar" session = boto3.session.Session() self.s3_client =session.client('s3', region_name="us-east-1") + logger.debug(f"bucket: {self.bucket_name}") def _get_bucket_uri(self, filename: str) -> str: @@ -52,10 +53,12 @@ class AWSBundleStorageClient(BundleStorageClient): self._download_to_file(uri, file) return file, uri else: + logger.info(f"{filename} does not exist") raise FileNotFound(filename) except Exception as e: logger.error(f"Failed to download file from {uri} {e}") + return None, None def upload_file(self, name: str, file: FileLikeObject) -> str: try: @@ -83,6 +86,7 @@ class AWSBundleStorageClient(BundleStorageClient): # fast operation no matter the size of the data object self.s3_client.head_object(Bucket=bucket_name, Key=object_name) except Exception as e: + logger.info(f"{uri} does not exist") return False return True diff --git a/app/bundles/storage.py b/app/bundles/storage.py index 0675ecebc7d33343ce5a6fc9b2a00f864eba8238..61d2b5e61465b063f13bcbc5c18cce4a0e5a6c7d 100644 --- a/app/bundles/storage.py +++ b/app/bundles/storage.py @@ -24,8 +24,8 @@ from osdu_api.providers.types import FileLikeObject import correlation import conf -_logger = logging.getLogger(__name__) -logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": "None"}) +logger = logging.getLogger(__name__) +#logger = correlation.DefaultExtrasAdapter(_logger, {"correlation_id": "None"}) class BundleStorageClient(abc.ABC): """Base interface for bundle storage clients""" @@ -33,13 +33,13 @@ class BundleStorageClient(abc.ABC): @abc.abstractmethod def download_file(self, filename: str, file: FileLikeObject) -> Tuple[FileLikeObject, str]: """Download file by name""" - #logger.info(f"download_file {filename}") + logger.debug(f"download_file {filename}") pass @abc.abstractmethod def upload_file(self, name: str, file: FileLikeObject) -> bool: """Upload file by name""" - #logger.info(f"upload_file {name}") + logger.debug(f"upload_file {name}") pass def _import_provider_specific_module(provider: str) -> str: @@ -62,5 +62,5 @@ def get_storage() -> BundleStorageClient: logger.critical(f"Error occurred while importing module for {cloud_env}") logger.critical(f"Exception: {exc}") - _logger.info(f"get_storage provider for cloud_env {cloud_env}") + logger.info(f"get_storage provider for cloud_env {cloud_env}") return provider_module.storage_client() diff --git a/app/conf.py b/app/conf.py index 0ae1c9ac37a8f2af5fb00d768483fec66556c92d..4a0603e1bd41eaff965775f1fbe055fc96373ab8 100644 --- a/app/conf.py +++ b/app/conf.py @@ -92,5 +92,12 @@ OPA_DATAAPI_CACHE_TTL=OPA_CACHE_TTL # RESPONSE_HEADERS_X_FRAME_OPTIONS = "DENY" # MOCK ENTITLEMENT - turns off security - only used for development purposes. Do Not Use -MOCK_ENTITLEMENT=False -MOCK_ENTITLEMENT_RESULT=[{'name': 'service.plugin.user', 'description': 'Datalake Plugin-Manager users', 'email': 'service.plugin.user@osdu.example.com'}, {'name': 'service.csv-parser.admin', 'description': 'Datalake csv-parser admins', 'email': 'service.csv-parser.admin@osdu.example.com'}, {'name': 'service.unit.admin', 'description': 'Datalake unit admins', 'email': 'service.unit.admin@osdu.example.com'}, {'name': 'service.edsdms.viewer', 'description': 'The viewer of the datalake edsdms service', 'email': 'service.edsdms.viewer@osdu.example.com'}, {'name': 'service.binarydms.viewer', 'description': 'The viewer of the datalake binarydms service', 'email': 'service.binarydms.viewer@osdu.example.com'}, {'name': 'service.schema-service.viewers', 'description': 'Viewers group for ddms schema service', 'email': 'service.schema-service.viewers@osdu.example.com'}, {'name': 'service.legal.admin', 'description': 'Datalake Legal admins', 'email': 'service.legal.admin@osdu.example.com'}, {'name': 'users.data.root', 'description': 'Datalake root users', 'email': 'users.data.root@osdu.example.com'}, {'name': 'data.test1', 'description': 'Storage integration test group', 'email': 'data.test1@osdu.example.com'}, {'name': 'users.datalake.admins', 'description': 'Datalake admins', 'email': 'users.datalake.admins@osdu.example.com'}, {'name': 'service.file.editors', 'description': 'Editors group for file service', 'email': 'service.file.editors@osdu.example.com'}, {'name': 'service.dataset.viewers', 'description': 'Viewers group for Dataset service', 'email': 'service.dataset.viewers@osdu.example.com'}, {'name': 'users', 'description': 'Datalake users', 'email': 'users@osdu.example.com'}, {'name': 'service.policy.admin', 'description': 'Datalake policy admins', 'email': 'service.policy.admin@osdu.example.com'}, {'name': 'service.binarydms.admin', 'description': 'Datalake binarydms admins', 'email': 'service.binarydms.admin@osdu.example.com'}, {'name': 'service.schema-service.admin', 'description': 'Admin group for ddms schema service', 'email': 'service.schema-service.admin@osdu.example.com'}, {'name': 'service.storage.creator', 'description': 'Datalake Storage creators', 'email': 'service.storage.creator@osdu.example.com'}, {'name': 'service.file.viewers', 'description': 'Viewers group for file service', 'email': 'service.file.viewers@osdu.example.com'}, {'name': 'service.workflow.viewer', 'description': 'Viewers group for workflow service', 'email': 'service.workflow.viewer@osdu.example.com'}, {'name': 'service.indexer.viewer', 'description': 'The viewer of the indexer service', 'email': 'service.indexer.viewer@osdu.example.com'}, {'name': 'service.legal.user', 'description': 'Datalake Legal users', 'email': 'service.legal.user@osdu.example.com'}, {'name': 'service.unit.creator', 'description': 'Datalake unit creators', 'email': 'service.unit.creator@osdu.example.com'}, {'name': 'cron.job', 'description': 'cron job role', 'email': 'cron.job@osdu.example.com'}, {'name': 'service.seismic-store.admin', 'description': 'Datalake seismic-store admins', 'email': 'service.seismic-store.admin@osdu.example.com'}, {'name': 'service.entitlements.admin', 'description': 'Datalake Entitlements admins', 'email': 'service.entitlements.admin@osdu.example.com'}, {'name': 'service.dataset.editors', 'description': 'Editors group for Dataset service', 'email': 'service.dataset.editors@osdu.example.com'}, {'name': 'service.workflow.creator', 'description': 'Creators group for workflow service', 'email': 'service.workflow.creator@osdu.example.com'}, {'name': 'service.search.user', 'description': 'Datalake Search users', 'email': 'service.search.user@osdu.example.com'}, {'name': 'service.unit.viewer', 'description': 'The viewer of the unit service', 'email': 'service.unit.viewer@osdu.example.com'}, {'name': 'data.default.owners', 'description': 'Default data owners', 'email': 'data.default.owners@osdu.example.com'}, {'name': 'service.workflow.admin', 'description': 'Admin group for workflow service', 'email': 'service.workflow.admin@osdu.example.com'}, {'name': 'service.entitlements.user', 'description': 'Datalake Entitlements users', 'email': 'service.entitlements.user@osdu.example.com'}, {'name': 'service.edsdms.user', 'description': 'Datalake edsdms user', 'email': 'service.edsdms.user@osdu.example.com'}, {'name': 'service.policy.user', 'description': 'The viewer of the datalake policy service', 'email': 'service.policy.user@osdu.example.com'}, {'name': 'service.ingest.viewer', 'description': 'The viewer of the ingest service', 'email': 'service.ingest.viewer@osdu.example.com'}, {'name': 'service.ingest.admin', 'description': 'Datalake ingest admins', 'email': 'service.ingest.admin@osdu.example.com'}, {'name': 'service.indexer.admin', 'description': 'Datalake indexer admins', 'email': 'service.indexer.admin@osdu.example.com'}, {'name': 'service.schema-service.editors', 'description': 'Editors group for ddms schema service', 'email': 'service.schema-service.editors@osdu.example.com'}, {'name': 'service.ingest.creator', 'description': 'Datalake ingest creators', 'email': 'service.ingest.creator@osdu.example.com'}, {'name': 'users.datalake.viewers', 'description': 'Datalake viewers', 'email': 'users.datalake.viewers@osdu.example.com'}, {'name': 'data.default.viewers', 'description': 'Default data viewers', 'email': 'data.default.viewers@osdu.example.com'}, {'name': 'service.search.admin', 'description': 'Datalake Search admins', 'email': 'service.search.admin@osdu.example.com'}, {'name': 'service.seismic-store.creator', 'description': 'Datalake seismic-store creators', 'email': 'service.seismic-store.creator@osdu.example.com'}, {'name': 'service.seismic-store.viewer', 'description': 'The viewer of the seismic-store service', 'email': 'service.seismic-store.viewer@osdu.example.com'}, {'name': 'users.datalake.editors', 'description': 'Datalake editors', 'email': 'users.datalake.editors@osdu.example.com'}, {'name': 'data.integration.test', 'description': 'Storage integration test group', 'email': 'data.integration.test@osdu.example.com'}, {'name': 'service.legal.editor', 'description': 'Datalake Legal editors', 'email': 'service.legal.editor@osdu.example.com'}, {'name': 'service.edsdms.admin', 'description': 'Datalake edsdms admins', 'email': 'service.edsdms.admin@osdu.example.com'}, {'name': 'service.messaging.user', 'description': 'Datalake Messaging users', 'email': 'service.messaging.user@osdu.example.com'}, {'name': 'service.csv-parser.creator', 'description': 'Datalake csv-parser creators', 'email': 'service.csv-parser.creator@osdu.example.com'}, {'name': 'service.delivery.viewer', 'description': 'Delivery role', 'email': 'service.delivery.viewer@osdu.example.com'}, {'name': 'service.storage.viewer', 'description': 'The viewer of the datalake storage service', 'email': 'service.storage.viewer@osdu.example.com'}, {'name': 'service.edsdms.creator', 'description': 'Datalake edsdms creators', 'email': 'service.edsdms.creator@osdu.example.com'}, {'name': 'service.storage.admin', 'description': 'Datalake Storage admins', 'email': 'service.storage.admin@osdu.example.com'}, {'name': 'service.binarydms.creator', 'description': 'Datalake binarydms creators', 'email': 'service.binarydms.creator@osdu.example.com'}, {'name': 'service.indexer.creator', 'description': 'Datalake indexer creators', 'email': 'service.indexer.creator@osdu.example.com'}, {'name': 'service.csv-parser.viewer', 'description': 'The viewer of the datalake csv-parser service', 'email': 'service.csv-parser.viewer@osdu.example.com'}, {'name': 'users.datalake.ops', 'description': 'Datalake ops', 'email': 'users.datalake.ops@osdu.example.com'}] \ No newline at end of file +MOCK_ENTITLEMENT = os.getenv("MOCK_ENTITLEMENT", 'False').lower() in ('true', '1', 't') +MOCK_ENTITLEMENT_RESULT=[{'name': 'service.plugin.user', 'description': 'Datalake Plugin-Manager users', 'email': 'service.plugin.user@osdu.example.com'}, {'name': 'service.csv-parser.admin', 'description': 'Datalake csv-parser admins', 'email': 'service.csv-parser.admin@osdu.example.com'}, {'name': 'service.unit.admin', 'description': 'Datalake unit admins', 'email': 'service.unit.admin@osdu.example.com'}, {'name': 'service.edsdms.viewer', 'description': 'The viewer of the datalake edsdms service', 'email': 'service.edsdms.viewer@osdu.example.com'}, {'name': 'service.binarydms.viewer', 'description': 'The viewer of the datalake binarydms service', 'email': 'service.binarydms.viewer@osdu.example.com'}, {'name': 'service.schema-service.viewers', 'description': 'Viewers group for ddms schema service', 'email': 'service.schema-service.viewers@osdu.example.com'}, {'name': 'service.legal.admin', 'description': 'Datalake Legal admins', 'email': 'service.legal.admin@osdu.example.com'}, {'name': 'users.data.root', 'description': 'Datalake root users', 'email': 'users.data.root@osdu.example.com'}, {'name': 'data.test1', 'description': 'Storage integration test group', 'email': 'data.test1@osdu.example.com'}, {'name': 'users.datalake.admins', 'description': 'Datalake admins', 'email': 'users.datalake.admins@osdu.example.com'}, {'name': 'service.file.editors', 'description': 'Editors group for file service', 'email': 'service.file.editors@osdu.example.com'}, {'name': 'service.dataset.viewers', 'description': 'Viewers group for Dataset service', 'email': 'service.dataset.viewers@osdu.example.com'}, {'name': 'users', 'description': 'Datalake users', 'email': 'users@osdu.example.com'}, {'name': 'service.policy.admin', 'description': 'Datalake policy admins', 'email': 'service.policy.admin@osdu.example.com'}, {'name': 'service.binarydms.admin', 'description': 'Datalake binarydms admins', 'email': 'service.binarydms.admin@osdu.example.com'}, {'name': 'service.schema-service.admin', 'description': 'Admin group for ddms schema service', 'email': 'service.schema-service.admin@osdu.example.com'}, {'name': 'service.storage.creator', 'description': 'Datalake Storage creators', 'email': 'service.storage.creator@osdu.example.com'}, {'name': 'service.file.viewers', 'description': 'Viewers group for file service', 'email': 'service.file.viewers@osdu.example.com'}, {'name': 'service.workflow.viewer', 'description': 'Viewers group for workflow service', 'email': 'service.workflow.viewer@osdu.example.com'}, {'name': 'service.indexer.viewer', 'description': 'The viewer of the indexer service', 'email': 'service.indexer.viewer@osdu.example.com'}, {'name': 'service.legal.user', 'description': 'Datalake Legal users', 'email': 'service.legal.user@osdu.example.com'}, {'name': 'service.unit.creator', 'description': 'Datalake unit creators', 'email': 'service.unit.creator@osdu.example.com'}, {'name': 'cron.job', 'description': 'cron job role', 'email': 'cron.job@osdu.example.com'}, {'name': 'service.seismic-store.admin', 'description': 'Datalake seismic-store admins', 'email': 'service.seismic-store.admin@osdu.example.com'}, {'name': 'service.entitlements.admin', 'description': 'Datalake Entitlements admins', 'email': 'service.entitlements.admin@osdu.example.com'}, {'name': 'service.dataset.editors', 'description': 'Editors group for Dataset service', 'email': 'service.dataset.editors@osdu.example.com'}, {'name': 'service.workflow.creator', 'description': 'Creators group for workflow service', 'email': 'service.workflow.creator@osdu.example.com'}, {'name': 'service.search.user', 'description': 'Datalake Search users', 'email': 'service.search.user@osdu.example.com'}, {'name': 'service.unit.viewer', 'description': 'The viewer of the unit service', 'email': 'service.unit.viewer@osdu.example.com'}, {'name': 'data.default.owners', 'description': 'Default data owners', 'email': 'data.default.owners@osdu.example.com'}, {'name': 'service.workflow.admin', 'description': 'Admin group for workflow service', 'email': 'service.workflow.admin@osdu.example.com'}, {'name': 'service.entitlements.user', 'description': 'Datalake Entitlements users', 'email': 'service.entitlements.user@osdu.example.com'}, {'name': 'service.edsdms.user', 'description': 'Datalake edsdms user', 'email': 'service.edsdms.user@osdu.example.com'}, {'name': 'service.policy.user', 'description': 'The viewer of the datalake policy service', 'email': 'service.policy.user@osdu.example.com'}, {'name': 'service.ingest.viewer', 'description': 'The viewer of the ingest service', 'email': 'service.ingest.viewer@osdu.example.com'}, {'name': 'service.ingest.admin', 'description': 'Datalake ingest admins', 'email': 'service.ingest.admin@osdu.example.com'}, {'name': 'service.indexer.admin', 'description': 'Datalake indexer admins', 'email': 'service.indexer.admin@osdu.example.com'}, {'name': 'service.schema-service.editors', 'description': 'Editors group for ddms schema service', 'email': 'service.schema-service.editors@osdu.example.com'}, {'name': 'service.ingest.creator', 'description': 'Datalake ingest creators', 'email': 'service.ingest.creator@osdu.example.com'}, {'name': 'users.datalake.viewers', 'description': 'Datalake viewers', 'email': 'users.datalake.viewers@osdu.example.com'}, {'name': 'data.default.viewers', 'description': 'Default data viewers', 'email': 'data.default.viewers@osdu.example.com'}, {'name': 'service.search.admin', 'description': 'Datalake Search admins', 'email': 'service.search.admin@osdu.example.com'}, {'name': 'service.seismic-store.creator', 'description': 'Datalake seismic-store creators', 'email': 'service.seismic-store.creator@osdu.example.com'}, {'name': 'service.seismic-store.viewer', 'description': 'The viewer of the seismic-store service', 'email': 'service.seismic-store.viewer@osdu.example.com'}, {'name': 'users.datalake.editors', 'description': 'Datalake editors', 'email': 'users.datalake.editors@osdu.example.com'}, {'name': 'data.integration.test', 'description': 'Storage integration test group', 'email': 'data.integration.test@osdu.example.com'}, {'name': 'service.legal.editor', 'description': 'Datalake Legal editors', 'email': 'service.legal.editor@osdu.example.com'}, {'name': 'service.edsdms.admin', 'description': 'Datalake edsdms admins', 'email': 'service.edsdms.admin@osdu.example.com'}, {'name': 'service.messaging.user', 'description': 'Datalake Messaging users', 'email': 'service.messaging.user@osdu.example.com'}, {'name': 'service.csv-parser.creator', 'description': 'Datalake csv-parser creators', 'email': 'service.csv-parser.creator@osdu.example.com'}, {'name': 'service.delivery.viewer', 'description': 'Delivery role', 'email': 'service.delivery.viewer@osdu.example.com'}, {'name': 'service.storage.viewer', 'description': 'The viewer of the datalake storage service', 'email': 'service.storage.viewer@osdu.example.com'}, {'name': 'service.edsdms.creator', 'description': 'Datalake edsdms creators', 'email': 'service.edsdms.creator@osdu.example.com'}, {'name': 'service.storage.admin', 'description': 'Datalake Storage admins', 'email': 'service.storage.admin@osdu.example.com'}, {'name': 'service.binarydms.creator', 'description': 'Datalake binarydms creators', 'email': 'service.binarydms.creator@osdu.example.com'}, {'name': 'service.indexer.creator', 'description': 'Datalake indexer creators', 'email': 'service.indexer.creator@osdu.example.com'}, {'name': 'service.csv-parser.viewer', 'description': 'The viewer of the datalake csv-parser service', 'email': 'service.csv-parser.viewer@osdu.example.com'}, {'name': 'users.datalake.ops', 'description': 'Datalake ops', 'email': 'users.datalake.ops@osdu.example.com'}] + +# Bootstrap +# Allow bootstrap API force=True to replace existing bundle with default +# This is useful: +# in automated testing to reset/start from a known state +# if someone breaks OPA by saving rego with syntax errors and it's not know which rego caused the issue +ALLOW_FORCE_BOOTSTRAP = os.getenv("ALLOW_FORCE_BOOTSTRAP", 'True').lower() in ('true', '1', 't') \ No newline at end of file diff --git a/app/main.py b/app/main.py index 2fd4a2102057fa7ecf43a78257285998d8fff159..edf823b295c63da89d6de6b3ba37632cf321b2cc 100644 --- a/app/main.py +++ b/app/main.py @@ -12,7 +12,7 @@ import os import coloredlogs import logging import conf -from api import policy_read_api, policy_update_api, policy_eval_api, health_api, compile_api, info_api, translate_api, diag_api, login_api, backup_api +from api import policy_read_api, policy_update_api, policy_eval_api, health_api, compile_api, info_api, translate_api, diag_api, login_api, backup_api, bootstrap_api from views import home import _buildinfo as b from auth import auth @@ -32,12 +32,20 @@ tags_metadata = [ }, { "name": "translate", - "description": "Translation APIs" + "description": "Translation APIs", + "externalDocs": { + "description": "Translate API documentation", + "url": "https://osdu.pages.opengroup.org/platform/security-and-compliance/policy/translate/" + } }, { "name": "diagnostic", "description": "Version information and other diagnostic APIs" }, + { + "name": "utility", + "description": "Service APIs" + }, { "name": "login", "description": "Login for Admin UI" @@ -49,7 +57,7 @@ tags_metadata = [ ] description = f""" - OSDU Policy Service v{b.version} API + OSDU Policy Service v{b.version} API for Milestone {b.milestone} Policy service is used for management and evaluation of dynamic policies in OSDU. @@ -62,6 +70,10 @@ Policy service has a runtime dependency on the following: * To read or evaluate policies, user calling the policy service, must be a member of _service.policy.user_ or _service.policy.admin_ group. * To create, update, or delete the policies, user must be a member of _service.policy.admin_ group. * User can be added to these groups by using entitlements service. + +### Useful information +* [Documentation](https://osdu.pages.opengroup.org/platform/security-and-compliance/policy/) +* [Source](https://community.opengroup.org/osdu/platform/security-and-compliance/policy) """ # setup loggers and make sure we can find logging.config @@ -130,15 +142,15 @@ async def policy_context_dependency(correlation_id: str = Header(None), user_age app = FastAPI( middleware=middleware, - title="Policy Service", + title="OSDU Policy Service", description=description, - version=b.version, + version=f"v{b.version} {b.milestone}", docs_url=conf.SERVICE_BASE_PATH + "/docs", redoc_url=conf.SERVICE_BASE_PATH + "/redoc", openapi_url=conf.SERVICE_BASE_PATH + "/openapi.json", contact={ "name": "OSDU Policy Service", - "url": "https://community.opengroup.org/osdu/platform/security-and-compliance/policy"}, + "url": "https://osdu.pages.opengroup.org/platform/security-and-compliance/policy/"}, license_info={ "name": "Apache 2.0", "url": "https://community.opengroup.org/osdu/platform/security-and-compliance/policy/-/raw/trusted-FastAPI/LICENSE", @@ -173,9 +185,10 @@ def configure_routing(): app.include_router(policy_eval_api.router, prefix=conf.SERVICE_BASE_PATH, tags=["evaluate"], dependencies=PROTECTED) app.include_router(translate_api.router, prefix=conf.SERVICE_BASE_PATH, tags=["translate"], dependencies=PROTECTED) app.include_router(info_api.router, prefix=conf.SERVICE_BASE_PATH, tags=["diagnostic"]) - app.include_router(compile_api.router, prefix=conf.SERVICE_BASE_PATH, tags=["diagnostic"], dependencies=PROTECTED) + app.include_router(compile_api.router, prefix=conf.SERVICE_BASE_PATH, tags=["utility"], dependencies=PROTECTED) app.include_router(health_api.router, prefix=conf.SERVICE_BASE_PATH, tags=["diagnostic"]) - app.include_router(backup_api.router, prefix=conf.SERVICE_BASE_PATH, tags=["diagnostic"], dependencies=PROTECTED) + app.include_router(backup_api.router, prefix=conf.SERVICE_BASE_PATH, tags=["utility"], dependencies=PROTECTED) + app.include_router(bootstrap_api.router, prefix=conf.SERVICE_BASE_PATH, tags=["utility"], dependencies=PROTECTED) if conf.ENABLE_ADMIN_UI: # only needed to support Admin UI diff --git a/app/tests/integration/test_integration_011_utility.py b/app/tests/integration/test_integration_011_utility.py new file mode 100644 index 0000000000000000000000000000000000000000..2a40958c07da860058a5cf578daf4c2ef500df2b --- /dev/null +++ b/app/tests/integration/test_integration_011_utility.py @@ -0,0 +1,61 @@ +# test_integration_011_utility.py +import pytest +# requires pytest_dependency to be installed +import unittest +import responses +from fastapi.testclient import TestClient +import requests +import re +import logging +import sys +import os +import base64 +import json +from pathlib import Path + +# so the directly in which you run pytest or unittest doesn't matter as much +sys.path.append(os.path.abspath('..')) +sys.path.append('tests') + +# local +from main import app +from auth import auth +import conf +import testlib + +# override dependency injection for authentication to entitlement service +from override_depends import override_require_authorized_user, set_authorize_session, ADMIN_ONLY_SVC, USER_AND_ADMIN_SVC, USER_ONLY_SVC, OTHER_ONLY_SVC + +TEST_DATA_DIR = Path(__file__).resolve().parent / 'data' + +client = TestClient(app) + +@pytest.mark.dependency(name="require_token") +def test_require_token(token): + assert token is not None, "No token provided on command line" + +def test_cloud_provider(cloud_provider): + """Check CLOUD_PROVIDER """ + assert cloud_provider is not None, "No CLOUD_PROVIDER env set" + +@pytest.mark.dependency(name="require_token") +def test_backup(token, data_partition, service_url, cloud_provider): + """ + Test Backup API + """ + url = service_url + conf.SERVICE_BASE_PATH + "/backup" + headers={'Authorization': 'Bearer ' + token, 'data-partition-id': data_partition} + r = requests.get(url, headers=headers) + if r.status_code != 200: + pytest.xfail(f"service_url /backup fail for {cloud_provider} {r.status_code} {r.text}") + +@pytest.mark.dependency(name="require_token") +def test_bootstrap(token, data_partition, service_url, cloud_provider): + """ + Test Bootstrap API + """ + url = service_url + conf.SERVICE_BASE_PATH + "/bootstrap" + headers={'Authorization': 'Bearer ' + token, 'data-partition-id': data_partition} + r = requests.post(url, headers=headers) + if r.status_code != 201 or r.status_code != 202 or r.status != 405 or r.status_code != 424: + pytest.xfail(f"service_url /bootstrap fail for {cloud_provider} {r.status_code} {r.text}") \ No newline at end of file diff --git a/docs/docs/bundles.md b/docs/docs/bundles.md index 483fc17f614a6bba151a92365fccb86ba8186b6b..d5488b8b30899291f1b3d4b65436ec7d22f4994f 100644 --- a/docs/docs/bundles.md +++ b/docs/docs/bundles.md @@ -59,28 +59,36 @@ Examples of the above can be found in [tests template directory](https://communi It is currently required to have backend CSP access and complete these two steps prior to adding a new partition or your new partition in OSDU will likely be unusable. -Two things must be done manually when adding a new partition to OSDU before running the add partition API in OSDU. +Two things must be done when adding a new partition to OSDU before (or after) running the add partition API in OSDU. -- Put a bundle in bundle server (S3 or other storage location) for the partition. This _bundle-`data-partition-id`.tar.gz_ should contain the following at a minimum: +### Create a bundle for partition + +Creating a bundle can be done two ways: + +- Calling the `/boostrap` Policy API + +- Update the backend storage directly. Put a bundle in bundle server (S3 or other storage location) for the partition. This _bundle-`data-partition-id`.tar.gz_ should contain the following at a minimum: * _.manifest_, * dataauthz, and * search +### Update OPA + - Update the [OPA configuration](https://www.openpolicyagent.org/docs/latest/configuration/) so that it knows it should read in the new bundle for the partition that will be created. To do this you need to provide OPA a new config.yaml. However this is CSP independent today, but in general can be done by updating the OPA config map. It is recommended to update your existing config map running in your kubernetes env. Once the config map in Kubernetes has been updated the OPA pods/service should be restarted to force the read-in of this configuration. Review OPA pod logs for errors after updating. -### Testing OPA configuration locally +#### Testing OPA configuration locally You can test the opa config using the OPA CLI. The configuration file can be in JSON or YAML format. Please note some documentation may refer to it as config.yaml or init.yaml - they mean the same thing. Not to be confused with kubenetes config map yaml - This is just the data config section of config map. ``` opa run -s -c config.yaml ``` -### Policy API for retrieving configuration +#### Policy API for retrieving configuration The Policy /diag/config [API](api.md) (which is likely disabled in production environments), allows you to retrieve the configuration detail of OPA. -### Automation for creating the deployed config map +#### Automation for creating the deployed config map Original template automation for config map: @@ -88,7 +96,7 @@ Original template automation for config map: * [Azure Config map](https://community.opengroup.org/osdu/platform/deployment-and-operations/helm-charts-azure/-/blob/master/osdu-azure/osdu-opa/templates/configmap-opa.yaml) * [GC](https://community.opengroup.org/osdu/platform/security-and-compliance/policy/-/blob/master/devops/gc/deploy/templates/opa-configmap.yaml) -### Review current config map +#### Review current config map If you have backend CSP access you can review the OPA config map: @@ -102,7 +110,7 @@ kubectl -n osdu-services describe configmap opa-config kubectl -n osdu-services get configmaps opa-config -o yaml > configmap.yaml ``` -### Updating config map +#### Updating config map An example opa_config: ``` @@ -149,13 +157,13 @@ You will need add the following section: All values for [min_delay_seconds and max_delay_seconds](https://www.openpolicyagent.org/docs/latest/configuration/#bundles) are supported, but you might need to adjust [BUNDLE_PAUSE](testing.md#bundle_pause) if you are running automated integration tests in this new data partition. -### Applying Update +#### Applying Update !!! warning You may not want to do this if your environment was created with cloudformation, helm or terraform. Use those methods to avoid conflicts. -#### Interactive update of config map via kubectl (not recommended) +##### Interactive update of config map via kubectl (not recommended) This can be tricky because of the spaces and newline \n in the data config section: ``` @@ -173,20 +181,20 @@ Interactive update example: kubectl -n osdu-services edit configmap opa-config ``` -#### Apply Modifications from file using kubectl +##### Apply Modifications from file using kubectl ``` kubectl apply -f configmap.yaml ``` -### Check for errors +#### Check for errors Example, your pod name and namespace may be different ``` kubectl -n osdu-services logs opa-agent-5ddf6fc476-lwp6p ``` -### Helm +#### Helm If your environment is managed by helm. Get RELEASE_NAME. ``` @@ -218,7 +226,7 @@ If no errors, for real: helm upgrade core core/core --reuse-values --version 0.19.1 -f opa-values.yaml ``` -### Terraform +#### Terraform OSDU M18 on AWS is expected to use Terraform for managing helm which controls the config map for OPA. When this is released these instructions will be updated. \ No newline at end of file diff --git a/docs/docs/install.md b/docs/docs/install.md index 5e3ecd1d13bc0d1b18e746aa1cbc254f2053c839..887a5d368769db4551291ba2f51213b07130163d 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -56,7 +56,8 @@ Currently supported values of `CLOUD_PROVIDER`: - `ENDPOINT_URL`, `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` - used by IBM (yes IBM). Please note only region `us-east-1` is currently supported. - `MINIO_ENDPOINT`, `MINIO_SECRET_KEY`, `MINIO_ACCESS_KEY` - used by Anthos (Reference Architecture). - `DISABLE_OPA_CACHE` - if set OPA caching will be disabled. -- `ENABLE_TRANSLATE_PREPROCESSOR` - by default True. If set to false it will revert search translate function to basically pre-M18 translate code. This would require all policies to be in a simple form (for example no deny). Included here for backwards testing only. It should never be disabled in qualification, preship testing, or production environments. +- `ENABLE_TRANSLATE_PREPROCESSOR` - by default True. If set to false it will revert search translate function to basically pre-M18 translate code. This would require all policies to be in a simple form (for example no deny). Included here for backwards compatibility testing only. It should never be disabled in qualification, preship testing, or production environments. +- `ALLOW_FORCE_BOOTSTRAP` - by default True. If set to false it will not allow you to revert bundle to default (if bundle exists). If true, the API will allow update to reset bundle to a known good state (useful for testing or when corrupt/bad rego policy files have been included in the bundle). ### Limit Number of Groups for Performance Reasons: diff --git a/docs/docs/testing.md b/docs/docs/testing.md index 031cbff7ebb666c33ba21dfc01b31e1c605d2026..06f62c149a0ec3703259c97f812025bda8610452 100644 --- a/docs/docs/testing.md +++ b/docs/docs/testing.md @@ -29,6 +29,7 @@ The [test directory](https://community.opengroup.org/osdu/platform/security-and- * `ENABLE_ADMIN_UI` with this turned on /adminui/ URL will be turned on within the container, which serves up /assets. This is added during build phase. When testing locally and not using containers it would be best to just run `npm start` * `--service_url` if you want to connect to a policy service other than default * Caching is now enabled for responses from OPA. This is configurable in conf.py +* conf.MOCK_ENTITLEMENT `MOCK_ENTITLEMENT` - Allows you to test and run policy without OSDU entitlement services ### BUNDLE_PAUSE