Commit 8c0ad4a4 authored by Marc Burnie [AWS]'s avatar Marc Burnie [AWS]
Browse files

Merge branch 'master' into master-dev-merge

parents be20cce7 007abda0
......@@ -29,7 +29,6 @@ variables:
IBM_TEST_SUBDIR: tests/ibm
IBM_OPA_BUNDLE_SERVER_BUCKET: opa-bundle-server-bucket
OSDU_GCP_ENABLE_BOOTSTRAP: "true"
OSDU_GCP_SERVICE: policy
OSDU_GCP_VENDOR: gcp
OSDU_GCP_OPA_CONFIG_SERVICE: opa-config
......@@ -42,8 +41,14 @@ variables:
OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap devops/gcp/opa devops/gcp/configmap_opa"
OSDU_GCP_HELM_CONFIG_SERVICE: policy-config
OSDU_GCP_HELM_DEPLOYMENT_SERVICE: policy-deploy
OSDU_GCP_HELM_OPA_CONFIG_SERVICE_VARS: "--set data.bucket_name=$OSDU_GCP_POLICY_BUCKET"
OSDU_GCP_HELM_OPA_CONFIG_SERVICE_VARS_DEV2: "--set data.bucket_name=$OSDU_GCP_POLICY_BUCKET"
# FIXME
OSDU_GCP_HELM_OPA_CONFIG_SERVICE_VARS: >-
--set data.bucket_name=$OSDU_GCP_POLICY_BUCKET
--set conf.data_partition_id=osdu
# FIXME
OSDU_GCP_HELM_OPA_CONFIG_SERVICE_VARS_DEV2: >-
--set data.bucket_name=$OSDU_GCP_POLICY_BUCKET
--set conf.data_partition_id=devtwo
OSDU_GCP_HELM_CONFIG_SERVICE_VARS: >-
--set data.log_level=INFO
--set data.opa_url=$OSDU_GCP_OPA_URL
......@@ -103,10 +108,12 @@ include:
osdu-gcp-containerize-bootstrap-gitlab:
variables:
OSDU_GCP_ENABLE_BOOTSTRAP: "true"
BUILD_PATH: devops/gcp/bootstrap-osdu-module/Dockerfile
osdu-gcp-containerize-bootstrap-gcr:
variables:
OSDU_GCP_ENABLE_BOOTSTRAP: "true"
BUILD_PATH: devops/gcp/bootstrap-osdu-module/Dockerfile
osdu-gcp-containerize-gitlab:
......
import os
import tarfile
from jinja2 import Environment, FileSystemLoader, select_autoescape
class BootstrapDataPartitionBundles:
def create_and_upload_dp_bundles(dp_id):
tar_name = "bundle-{dp}.tar.gz".format(dp=dp_id)
dataauthz_template_name = "dataauthz_template.rego"
manifest_template_name = "manifest_template.manifest"
dataauthz_filename = "dataauthz.rego"
manifest_filename = ".manifest"
template_path = "devops/gcp/bootstrap-osdu-module/templates/"
env = Environment(
loader=FileSystemLoader(template_path),
autoescape=select_autoescape()
)
dataauthz_template = env.get_template(dataauthz_template_name)
manifest_template = env.get_template(manifest_template_name)
dataauthz_render = dataauthz_template.render(dp_id=dp_id)
manifest_render = manifest_template.render(dp_id=dp_id)
with open(dataauthz_filename,"w") as f1:
f1.write(dataauthz_render)
with open(manifest_filename, "w") as f2:
f2.write(manifest_render)
with tarfile.open(tar_name, "w:gz") as tar_handle:
tar_handle.add(os.path.abspath(dataauthz_filename), arcname=dataauthz_filename)
tar_handle.add(os.path.abspath(manifest_filename), arcname=manifest_filename)
# Initialize class and upload bundles
if __name__ == '__main__':
BootstrapDataPartitionBundles.create_and_upload_dp_bundles(os.environ.get('DATA_PARTITION'))
......@@ -5,7 +5,9 @@ COPY . /opt/
RUN chmod 775 /opt/bootstrap_policy.sh
RUN pip3 install -r /opt/requirements_bootstrap.txt
RUN pip3 install -r /opt/requirements_bootstrap.txt
RUN pip3 install -r /opt/devops/gcp/bootstrap-osdu-module/requirements.txt
CMD ["/bin/bash", "-c", "source /opt/bootstrap_policy.sh"]
......@@ -7,6 +7,8 @@ export BEARER_TOKEN=`gcloud auth print-identity-token --audiences=${AUDIENCES}`
echo "Achive bundle of policies and push to bucket"
tar -czf bundle.tar.gz --directory='./opt/deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose
mkdir --parents ./opt/policies ; mv bundle.tar.gz $_
python3 /opt/devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py
mv /opt/devops/gcp/bootstrap-osdu-module/bundle-${DATA_PARTITION}.tar.gz ./opt/policies
gsutil rsync ./opt/policies gs://${POLICY_BUCKET}/
echo "Achive bundle of policies and push to bucket - DONE!"
......
jinja2
\ No newline at end of file
package osdu.partition["{{dp_id}}"].dataauthz
import data.osdu.instance.dataauthz as centralauthz
records := centralauthz.records
{
"roots": ["osdu/partition/{{dp_id}}"]
}
\ No newline at end of file
......@@ -12,3 +12,5 @@ data:
ENTITLEMENTS_BASE_PATH: "{{ .Values.data.entitlements_base_path }}"
LEGAL_BASE_URL: "{{ .Values.data.legal_base_url }}"
POLICY_BUCKET: "{{ .Values.data.bucket_name }}"
USE_BUNDLES: "{{ .Values.data.use_bundles }}"
CLOUD_PROVIDER: "{{ .Values.data.cloud_provider }}"
......@@ -5,6 +5,8 @@ data:
entitlements_base_path: "/api/entitlements/v2/groups"
legal_base_url: "http://legal"
bucket_name: ""
use_bundles: "yes"
cloud_provider: "gcp"
conf:
configmap: "policy-config"
......
......@@ -16,7 +16,10 @@ data:
- "{{ .Values.data.scopes }}"
bundles:
authz:
osdu/instance:
service: gcs
# NOTE ?alt=media is required
resource: 'bundle.tar.gz?alt=media'
osdu/partition/{{ .Values.conf.data_partition_id }}:
service: gcs
resource: 'bundle-{{ .Values.conf.data_partition_id }}.tar.gz?alt=media'
......@@ -5,3 +5,4 @@ data:
conf:
configmap: "opa-config"
app_name: "opa"
data_partition_id: ""
......@@ -40,6 +40,11 @@ spec:
volumeMounts:
- mountPath: /config
name: "{{ .Values.conf.configmap }}"
env:
- name: ENTITLEMENTS_BASE_URL
value: "http://entitlements"
- name: LEGAL_BASE_URL
value: "http://legal"
volumes:
- name: "{{ .Values.conf.configmap }}"
configMap:
......
......@@ -140,9 +140,12 @@ osdu-gcp-bootstrap:
script:
- *common_part
- >
- export BEARER_TOKEN=`gcloud auth print-identity-token --audiences=$GOOGLE_AUDIENCE`
- pip install -r requirements.txt
- python3 deployment/scripts/BootstrapDefaultPolicies.py -u $OSDU_GCP_POLICY_URL
- pip install -r requirements_bootstrap.txt
- tar -czf bundle.tar.gz ./deployment/default-policies/.manifest ./deployment/default-policies/dataauthz.rego ./deployment/default-policies/entitlements.rego ./deployment/default-policies/legal.rego
- mkdir --parents ./policies ; mv bundle.tar.gz $_
- python3 devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py
- mv bundle-$DATA_PARTITION.tar.gz ./policies
- gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/
rules:
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/'
when: never
......@@ -160,8 +163,11 @@ osdu-gcp-bootstrap-bundle:
script:
- *common_part
- >
- tar -czf bundle.tar.gz --directory='./deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose
- pip install -r requirements_bootstrap.txt
- tar -czf bundle.tar.gz ./deployment/default-policies/.manifest ./deployment/default-policies/dataauthz.rego ./deployment/default-policies/entitlements.rego ./deployment/default-policies/legal.rego
- mkdir --parents ./policies ; mv bundle.tar.gz $_
- python3 devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py
- mv bundle-$DATA_PARTITION.tar.gz ./policies
- gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/
rules:
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/'
......@@ -180,8 +186,11 @@ osdu-gcp-dev2-bootstrap-bundle:
script:
- *common_part
- >
- tar -czf bundle.tar.gz --directory='./deployment/default-policies' --exclude='./bootstrap_sequence.json' . --verbose
- pip install -r requirements_bootstrap.txt
- tar -czf bundle.tar.gz ./deployment/default-policies/.manifest ./deployment/default-policies/dataauthz.rego ./deployment/default-policies/entitlements.rego ./deployment/default-policies/legal.rego
- mkdir --parents ./policies ; mv bundle.tar.gz $_
- python3 devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py
- mv bundle-$DATA_PARTITION.tar.gz ./policies
- gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/
rules:
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/'
......@@ -198,9 +207,12 @@ osdu-gcp-dev2-bootstrap:
script:
- *common_part
- >
- export BEARER_TOKEN=`gcloud auth print-identity-token --audiences=$GOOGLE_AUDIENCE`
- pip install -r requirements.txt
- python3 deployment/scripts/BootstrapDefaultPolicies.py -u $OSDU_GCP_POLICY_URL
- pip install -r requirements_bootstrap.txt
- tar -czf bundle.tar.gz ./deployment/default-policies/.manifest ./deployment/default-policies/dataauthz.rego ./deployment/default-policies/entitlements.rego ./deployment/default-policies/legal.rego
- mkdir --parents ./policies ; mv bundle.tar.gz $_
- python3 devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py
- mv bundle-$DATA_PARTITION.tar.gz ./policies
- gsutil rsync policies gs://$OSDU_GCP_POLICY_BUCKET/
rules:
- if: '$OSDU_GCP == "true" && $CI_COMMIT_BRANCH =~ /^release/'
when: on_success
......
{
"info": {
"_postman_id": "58acbe53-e7fa-40c6-b37a-78a0c9295f6a",
"name": "Bundle based dynamic policies tests",
"schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json"
},
"item": [
{
"name": "Policy service",
"item": [
{
"name": "All default policies",
"request": {
"method": "GET",
"header": [
{
"key": "Authorization",
"value": "Bearer {{request_token}}",
"type": "text"
},
{
"key": "data-partition-id",
"value": "{{data_partition_id}}",
"type": "text"
}
],
"url": {
"raw": "{{osdu_environment_url}}/api/policy/v1/policies",
"host": [
"{{osdu_environment_url}}"
],
"path": [
"api",
"policy",
"v1",
"policies"
]
}
},
"response": []
},
{
"name": "Default instance policy",
"request": {
"method": "GET",
"header": [
{
"key": "Authorization",
"value": "Bearer {{request_token}}",
"type": "text"
},
{
"key": "data-partition-id",
"value": "{{data_partition_id}}",
"type": "text"
}
],
"url": {
"raw": "{{osdu_environment_url}}/api/policy/v1/policies/osdu/instance/dataauthz.rego",
"host": [
"{{osdu_environment_url}}"
],
"path": [
"api",
"policy",
"v1",
"policies",
"osdu",
"instance",
"dataauthz.rego"
]
}
},
"response": []
},
{
"name": "Default partition policy",
"request": {
"method": "GET",
"header": [
{
"key": "Authorization",
"value": "Bearer {{request_token}}",
"type": "text"
},
{
"key": "data-partition-id",
"value": "{{data_partition_id}}",
"type": "text"
}
],
"url": {
"raw": "{{osdu_environment_url}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/dataauthz.rego",
"host": [
"{{osdu_environment_url}}"
],
"path": [
"api",
"policy",
"v1",
"policies",
"osdu",
"partition",
"{{data_partition_id}}",
"dataauthz.rego"
]
}
},
"response": []
},
{
"name": "Create partition policy",
"request": {
"method": "PUT",
"header": [
{
"key": "Authorization",
"value": "Bearer {{request_token}}",
"type": "text"
},
{
"key": "data-partition-id",
"value": "{{data_partition_id}}",
"type": "text"
}
],
"body": {
"mode": "raw",
"raw": "package osdu.partition[\"{{data_partition_id}}\"].mytest\n\nallow = false"
},
"url": {
"raw": "{{osdu_environment_url}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/mytest.rego",
"host": [
"{{osdu_environment_url}}"
],
"path": [
"api",
"policy",
"v1",
"policies",
"osdu",
"partition",
"{{data_partition_id}}",
"mytest.rego"
]
}
},
"response": []
},
{
"name": "Created policy",
"request": {
"method": "GET",
"header": [
{
"key": "Authorization",
"value": "Bearer {{request_token}}",
"type": "text"
},
{
"key": "data-partition-id",
"value": "{{data_partition_id}}",
"type": "text"
}
],
"url": {
"raw": "{{osdu_environment_url}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/mytest.rego",
"host": [
"{{osdu_environment_url}}"
],
"path": [
"api",
"policy",
"v1",
"policies",
"osdu",
"partition",
"{{data_partition_id}}",
"mytest.rego"
]
}
},
"response": []
},
{
"name": "Delete created policy",
"request": {
"method": "DELETE",
"header": [
{
"key": "Authorization",
"value": "Bearer {{request_token}}",
"type": "text"
},
{
"key": "data-partition-id",
"value": "{{data_partition_id}}",
"type": "text"
}
],
"url": {
"raw": "{{osdu_environment_url}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/mytest.rego",
"host": [
"{{osdu_environment_url}}"
],
"path": [
"api",
"policy",
"v1",
"policies",
"osdu",
"partition",
"{{data_partition_id}}",
"mytest.rego"
]
}
},
"response": []
}
]
},
{
"name": "Storage integration",
"item": [
{
"name": "Can create record with access",
"request": {
"method": "PUT",
"header": [
{
"key": "Authorization",
"value": "Bearer {{request_token}}",
"type": "text"
},
{
"key": "data-partition-id",
"value": "{{data_partition_id}}",
"type": "text"
}
],
"body": {
"mode": "raw",
"raw": "[{\n \"kind\": \"{{data_partition_id}}:osdu:well-master:0.2.0\",\n \"legal\": {\n \"legaltags\": [\n \"{{data_partition_id}}-public-usa-dataset-1\"\n ],\n \"otherRelevantDataCountries\": [\n \"US\"\n ]\n },\n \"acl\": {\n \"owners\": [\n \"data.default.owner@{{data_partition_id}}.{{entitlements_group_domain}}\"\n ],\n \"viewers\": [\n \"data.default.viewer@{{data_partition_id}}.{{entitlements_group_domain}}\"\n ]\n },\n \"id\": \"{{data_partition_id}}:doc:dynamic-policy-test-data-2\",\n \"data\": {\n \"description\": \"Dynamic policy test record 2\"\n }\n }]",
"options": {
"raw": {
"language": "json"
}
}
},
"url": {
"raw": "{{osdu_environment_url}}/api/storage/v2/records",
"host": [
"{{osdu_environment_url}}"
],
"path": [
"api",
"storage",
"v2",
"records"
]
}
},
"response": []
},
{
"name": "Cannot create record without acl membership",
"request": {
"method": "PUT",
"header": [
{
"key": "Authorization",
"value": "Bearer {{request_token}}",
"type": "text"
},
{
"key": "data-partition-id",
"value": "{{data_partition_id}}",
"type": "text"
}
],
"body": {
"mode": "raw",
"raw": "[{\n \"kind\": \"{{data_partition_id}}:osdu:well-master:0.2.0\",\n \"legal\": {\n \"legaltags\": [\n \"{{data_partition_id}}-public-usa-dataset-1\"\n ],\n \"otherRelevantDataCountries\": [\n \"US\"\n ]\n },\n \"acl\": {\n \"owners\": [\n \"data.notdefault.owner@{{data_partition_id}}.{{entitlements_group_domain}}\"\n ],\n \"viewers\": [\n \"data.default.viewer@{{data_partition_id}}.{{entitlements_group_domain}}\"\n ]\n },\n \"id\": \"{{data_partition_id}}:doc:dynamic-policy-test-data-1\",\n \"data\": {\n \"description\": \"Dynamic policy test record 1\"\n }\n }]",
"options": {
"raw": {
"language": "json"
}
}
},
"url": {
"raw": "{{osdu_environment_url}}/api/storage/v2/records",
"host": [
"{{osdu_environment_url}}"
],
"path": [
"api",
"storage",
"v2",
"records"
]
}
},
"response": []
},
{
"name": "Change partition policy to allow all",
"request": {
"method": "PUT",
"header": [
{
"key": "Authorization",
"value": "Bearer {{request_token}}",
"type": "text"
},
{
"key": "data-partition-id",
"value": "{{data_partition_id}}",
"type": "text"
}
],
"body": {
"mode": "raw",
"raw": "package osdu.partition[\"{{data_partition_id}}\"].dataauthz\n\nrecords[response] {\n id := input.records[_].id\n errors := []\n\n response := {\n \"id\" : id,\n \"errors\" : errors\n }\n}\n"
},
"url": {
"raw": "{{osdu_environment_url}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/dataauthz.rego",
"host": [
"{{osdu_environment_url}}"
],
"path": [
"api",
"policy",
"v1",
"policies",
"osdu",
"partition",
"{{data_partition_id}}",
"dataauthz.rego"
]
}
},
"response": []
},
{
"name": "Changed policy loaded",
"request": {
"method": "GET",
"header": [
{
"key": "Authorization",
"value": "Bearer {{request_token}}",
"type": "text"
},
{
"key": "data-partition-id",
"value": "{{data_partition_id}}",
"type": "text"
}
],
"url": {
"raw": "{{osdu_environment_url}}/api/policy/v1/policies/osdu/partition/{{data_partition_id}}/dataauthz.rego",
"host": [
"{{osdu_environment_url}}"
],
"path": [
"api",
"policy",