Merge branch 'opa_check' into 'master'

admincli check command, NOTICE update, CI/CD fix for AWS and test improvement

See merge request !232

NOTICE

@@ -60,6 +60,7 @@ The following software have components provided under the terms of this license:
 - Jinja2 (from http://jinja.pocoo.org/, https://palletsprojects.com/p/jinja/)
 - MarkupSafe (from https://palletsprojects.com/p/markupsafe/)
 - Werkzeug (from https://palletsprojects.com/p/werkzeug/)
+- asgiref (from https://github.com/django/asgiref/)
 - click (from http://github.com/mitsuhiko/click, https://palletsprojects.com/p/click/)
 - cryptography (from https://github.com/pyca/cryptography)
 - httpcore (from https://github.com/encode/httpcore)
@@ -77,7 +78,7 @@ The following software have components provided under the terms of this license:
 - python-dateutil (from https://github.com/dateutil/dateutil)
 - sniffio (from https://github.com/python-trio/sniffio)
 - starlette (from https://github.com/encode/starlette, https://pypi.org/project/starlette/0.21.0/)
-- uvicorn (from https://github.com/tomchristie/uvicorn, https://pypi.org/project/uvicorn/0.18.3/, https://www.uvicorn.org/)
+- uvicorn (from https://github.com/tomchristie/uvicorn, https://pypi.org/project/uvicorn/0.18.3/, https://pypi.org/project/uvicorn/0.19.0/, https://www.uvicorn.org/)
 
 ========================================================================
 CC-BY-2.5
@@ -92,8 +93,6 @@ CC-BY-3.0
 ========================================================================
 CC0-1.0
 ========================================================================
-The following software have components provided under the terms of this license:
-
 - ecdsa (from http://github.com/tlsfuzzer/python-ecdsa)
 
 ========================================================================
@@ -149,7 +148,7 @@ The following software have components provided under the terms of this license:
 - Flask (from https://palletsprojects.com/p/flask)
 - PyJWT (from http://github.com/jpadilla/pyjwt, https://github.com/jpadilla/pyjwt)
 - PyYAML (from http://pyyaml.org/wiki/PyYAML)
-- anyio (from https://pypi.org/project/anyio/3.3.0/, https://pypi.org/project/anyio/3.6.1/)
+- anyio (from https://pypi.org/project/anyio/3.3.0/, https://pypi.org/project/anyio/3.6.1/, https://pypi.org/project/anyio/3.6.2/)
 - attrs (from https://attrs.readthedocs.io/, https://www.attrs.org/)
 - azure-common (from https://github.com/Azure/azure-sdk-for-python)
 - azure-core (from https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/core/azure-core)
@@ -159,10 +158,13 @@ The following software have components provided under the terms of this license:
 - botocore (from https://github.com/boto/botocore)
 - cachetools (from https://github.com/tkem/cachetools/)
 - cffi
+- coloredlogs (from https://coloredlogs.readthedocs.io)
 - coverage (from https://github.com/nedbat/coveragepy)
 - ecdsa (from http://github.com/tlsfuzzer/python-ecdsa)
-- fastapi (from https://pypi.org/project/fastapi/0.85.0/)
+- exceptiongroup (from https://pypi.org/project/exceptiongroup/1.0.0rc9/)
+- fastapi (from https://pypi.org/project/fastapi/0.85.0/, https://pypi.org/project/fastapi/0.85.1/)
 - h11
+- humanfriendly (from https://humanfriendly.readthedocs.io)
 - iniconfig (from http://github.com/RonnyPfannschmidt/iniconfig)
 - jmespath (from https://github.com/jmespath/jmespath.py)
 - jsonschema
@@ -172,7 +174,7 @@ The following software have components provided under the terms of this license:
 - munch (from http://github.com/Infinidat/munch)
 - pluggy
 - py (from https://py.readthedocs.io/)
-- pydantic (from https://github.com/pydantic/pydantic)
+- pydantic (from https://github.com/pydantic/pydantic, https://github.com/samuelcolvin/pydantic)
 - pyparsing (from http://pyparsing.wikispaces.com/)
 - pyrsistent (from http://github.com/tobgu/pyrsistent/)
 - pytest (from http://pytest.org, https://docs.pytest.org/en/latest/)
@@ -185,9 +187,12 @@ The following software have components provided under the terms of this license:
 - requests (from http://python-requests.org, https://requests.readthedocs.io)
 - six (from http://pypi.python.org/pypi/six/, https://github.com/benjaminp/six)
 - sniffio (from https://github.com/python-trio/sniffio)
+- starlette (from https://pypi.org/project/starlette/0.21.0/)
+- starlette-context (from https://github.com/tomwojcik/starlette-context)
 - toml (from https://github.com/uiri/toml)
 - tomli (from https://pypi.org/project/tomli/1.2.2/, https://pypi.org/project/tomli/2.0.0/, https://pypi.org/project/tomli/2.0.1/)
 - urllib3 (from https://urllib3.readthedocs.io/)
+- uuid7 (from https://github.com/stevesimmons/uuid7)
 
 ========================================================================
 MIT-CMU
@@ -201,6 +206,7 @@ MPL-2.0
 ========================================================================
 The following software have components provided under the terms of this license:
 
+- botocore (from https://github.com/boto/botocore)
 - certifi (from https://github.com/certifi/python-certifi)
 
 ========================================================================
@@ -215,6 +221,7 @@ Python-2.0
 ========================================================================
 The following software have components provided under the terms of this license:
 
+- exceptiongroup (from https://pypi.org/project/exceptiongroup/1.0.0rc9/)
 - fastapi (from https://pypi.org/project/fastapi/0.85.0/)
 - portalocker (from https://github.com/WoLpH/portalocker)
 - pytest-mock (from https://github.com/pytest-dev/pytest-mock/)

frontend/admincli/opa.py

+import os
 import requests
+import subprocess
+import tempfile
+
+class OpaCheck:
+    fpath = None
+    def __init__(self):
+        self.tempdir = tempfile.mkdtemp()
+
+    def createfile(self, rego: str, filename: str):
+        self.fpath = os.path.join(self.tempdir, filename)
+        f = open(self.fpath, "w")
+        f.write(rego)
+        f.close()
+
+    def opa_check_file(self):
+        try:
+            result = subprocess.run(['opa', 'check', self.fpath], capture_output=True, text=True)
+        except FileNotFoundError as err:
+            os.remove(self.fpath)
+            return(-1,"OPA Binary not found")
+        os.remove(self.fpath)
+        return(result.returncode, result.stderr)
+
+    def check(self, rego: str, filename: str):
+        self.createfile(rego, filename)
+        return self.opa_check_file()
+
+    def delete(self):
+        os.rmdir(self.tempdir)
 
 def put_opa_policy_direct(policy_id, data, base_url, timeout=20):
     """

frontend/admincli/pol.py

@@ -879,6 +879,56 @@ def add_to_opa(ctx: typer.Context,
         error_console.print(f"Error adding policy {policy_id} to OPA {r.json()} {r.status_code}")
         raise typer.Exit(1) # non-zero exit status
 
+@cli.command("check", rich_help_panel="Policy Developer Utils")
+def opa_check(
+        ctx: typer.Context,
+        file_list: List[str]= typer.Argument(...),
+        display: bool = typer.Option(False, "--display", "-d", help="Display policy"),
+        template: bool = typer.Option(False, "--template", "-t", help="Input is a template to be rendered"),
+    ):
+    """
+    Check rego file for errors
+    """
+    if file_list == ['-']:
+        file_list = sys.stdin.read().strip().split()
+    
+    chk = opa.OpaCheck()
+    for file in file_list:
+        policy_id = os.path.basename(file)
+        console.print(f"Reading contents of {file} as '{policy_id}'")
+
+        with rich.progress.open(file, "r") as f:
+            data = f.read()
+        f.close()
+
+        if template:
+            try:
+                template_data = Template(data)
+                data = template_data.substitute(
+                    {
+                        'data_partition': ctx.obj.data_partition,
+                        'DATA_PARTITION': ctx.obj.data_partition,
+                        'name': policy_id.removesuffix('.rego')
+                    }
+                )
+            except ValueError as err:
+                error_console.print(f"Error in {file}: {err}")
+        if display:
+            console.print(Panel(data, title=f"{policy_id}", highlight=True))
+
+        with console.status(f"Checking {policy_id}"):
+            result, error_msg = chk.check(rego=data, filename=policy_id)
+        if result:
+            if 'rego_parse_error' in error_msg:
+                error_console.print(f"{policy_id}: rego_parse_error:\n{error_msg}")
+            else:
+                error_console.print(f"{policy_id}: error {error_msg}")
+            chk.delete() # cleanup before exit
+            raise typer.Exit(1) # non-zero exit status
+        else:
+            console.print(f"{policy_id}: [green]OK[/]")
+    chk.delete()
+
 @cli.command("opa-rm", rich_help_panel="Policy Developer Utils")
 def delete_from_opa(
         ctx: typer.Context,

frontend/admincli/requirements-dev.txt

@@ -4,3 +4,4 @@ requests == 2.25.1
 rich == 12.6.0
 typer == 0.6.1
 uuid7 == 0.1.0
+boto3

frontend/admincli/tests/test_cli.py

@@ -181,6 +181,9 @@ def test_cli_legaltags_random():
 def test_cli_policy_eval():
     legal_result = runner.invoke(cli, ["legal-tags", "--random"])
     legal_tag = legal_result.stdout.strip()
+    result = runner.invoke(cli, ["ls", "search2", "--quiet"])
+    if result.exit_code:
+        pytest.xfail(f"Policy search2 not available yet")
     result = runner.invoke(cli, ["eval", "--output=simple", "-f", "tests/evaluate_query.json", "-t", "search2", f"--legal-tag={legal_tag}", "--force"])
     assert result.exit_code == 0, f"expected 0 exit status. stdout: {result.stdout}"
     assert "{}" in result.stdout