From 12983fdd5d6f323385dc3a7784929dcf67f0e4df Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Sat, 17 Sep 2022 10:44:40 -0400 Subject: [PATCH 01/10] attempt to fix gcp integration tests --- devops/gcp/pipeline/override-stages.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index 43c109d..c2c1f57 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -244,7 +244,8 @@ osdu-gcp-test-python: - export BEARER_TOKEN=`gcloud auth print-access-token` - pip install -r requirements.txt - pip install -r requirements_dev.txt - - cd tests/integration + - cd app + - pip install -r requirements.txt - python3 -m pytest --token=$BEARER_TOKEN --service_url=$OSDU_GCP_POLICY_URL --data_partition=$DATA_PARTITION osdu-gcp-dev2-test-python: @@ -257,7 +258,8 @@ osdu-gcp-dev2-test-python: - export BEARER_TOKEN=`gcloud auth print-access-token` - pip install -r requirements.txt - pip install -r requirements_dev.txt - - cd tests/integration + - cd app + - pip install -r requirements.txt - python3 -m pytest --token=$BEARER_TOKEN --service_url=$OSDU_GCP_POLICY_URL --data_partition=$DATA_PARTITION osdu-gcp-containerize-bootstrap-gitlab: -- GitLab From 478903c27d9e35b691e69dc10a22e3a19d2f47c9 Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Sat, 17 Sep 2022 12:09:27 -0400 Subject: [PATCH 02/10] move to port 8080 --- devops/gcp/deploy/templates/service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/gcp/deploy/templates/service.yaml b/devops/gcp/deploy/templates/service.yaml index 38dba1e..9a94a8d 100644 --- a/devops/gcp/deploy/templates/service.yaml +++ b/devops/gcp/deploy/templates/service.yaml @@ -12,7 +12,7 @@ spec: ports: - protocol: TCP port: 80 - targetPort: 80 + targetPort: 8080 name: http selector: app: "{{ .Values.conf.appName }}" -- GitLab From 8bd9d773a920e3a8afc5b1807ec53c3a8577c24d Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Mon, 19 Sep 2022 10:39:09 -0400 Subject: [PATCH 03/10] Dockerfile updates --- build/Dockerfile | 19 +++++++++++++------ build/aws/build-aws/Dockerfile | 23 ++++++++++++++--------- build/azure/Dockerfile | 19 +++++++++++++------ build/gcp/Dockerfile | 19 +++++++++++++------ 4 files changed, 53 insertions(+), 27 deletions(-) diff --git a/build/Dockerfile b/build/Dockerfile index bbe8000..566b9cc 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -1,10 +1,17 @@ -FROM python:3.7-slim-buster +# This Dockerfile assumes you used "make build" to convert the AdminUI to static files in frontend/adminui/dist/policyservice +FROM python:3.9-slim-buster -WORKDIR /opt - -COPY ./ /opt +# set environment variables +# PYTHONDONTWRITEBYTECODE - Prevents Python from writing pyc files to disc (equivalent to python -B option) +ENV PYTHONDONTWRITEBYTECODE 1 +# PYTHONUNBUFFERED - Prevents Python from buffering stdout and stderr (equivalent to python -u option) +ENV PYTHONUNBUFFERED 1 -RUN pip install -r requirements.txt +WORKDIR /opt +COPY ./app /opt +RUN pip install --no-cache-dir --upgrade -r /opt/requirements.txt -ENTRYPOINT python ./server.py +# Admin UI POC +#COPY ./frontend/adminui/dist/policyservice /opt/assets +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/build/aws/build-aws/Dockerfile b/build/aws/build-aws/Dockerfile index e1f85c5..566b9cc 100644 --- a/build/aws/build-aws/Dockerfile +++ b/build/aws/build-aws/Dockerfile @@ -1,12 +1,17 @@ -FROM python:3.7-slim-buster -WORKDIR /opt +# This Dockerfile assumes you used "make build" to convert the AdminUI to static files in frontend/adminui/dist/policyservice +FROM python:3.9-slim-buster + +# set environment variables +# PYTHONDONTWRITEBYTECODE - Prevents Python from writing pyc files to disc (equivalent to python -B option) +ENV PYTHONDONTWRITEBYTECODE 1 +# PYTHONUNBUFFERED - Prevents Python from buffering stdout and stderr (equivalent to python -u option) +ENV PYTHONUNBUFFERED 1 -COPY ./ /opt +WORKDIR /opt +COPY ./app /opt +RUN pip install --no-cache-dir --upgrade -r /opt/requirements.txt +# Admin UI POC +#COPY ./frontend/adminui/dist/policyservice /opt/assets -RUN pip install -r requirements.txt -RUN apt-get update && apt-get install curl -y -RUN curl -L -o opa https://openpolicyagent.org/downloads/latest/opa_linux_amd64_static -RUN chmod 755 ./opa -RUN mkdir /policies -CMD python ./server.py \ No newline at end of file +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/build/azure/Dockerfile b/build/azure/Dockerfile index bbe8000..566b9cc 100644 --- a/build/azure/Dockerfile +++ b/build/azure/Dockerfile @@ -1,10 +1,17 @@ -FROM python:3.7-slim-buster +# This Dockerfile assumes you used "make build" to convert the AdminUI to static files in frontend/adminui/dist/policyservice +FROM python:3.9-slim-buster -WORKDIR /opt - -COPY ./ /opt +# set environment variables +# PYTHONDONTWRITEBYTECODE - Prevents Python from writing pyc files to disc (equivalent to python -B option) +ENV PYTHONDONTWRITEBYTECODE 1 +# PYTHONUNBUFFERED - Prevents Python from buffering stdout and stderr (equivalent to python -u option) +ENV PYTHONUNBUFFERED 1 -RUN pip install -r requirements.txt +WORKDIR /opt +COPY ./app /opt +RUN pip install --no-cache-dir --upgrade -r /opt/requirements.txt -ENTRYPOINT python ./server.py +# Admin UI POC +#COPY ./frontend/adminui/dist/policyservice /opt/assets +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/build/gcp/Dockerfile b/build/gcp/Dockerfile index bbe8000..566b9cc 100644 --- a/build/gcp/Dockerfile +++ b/build/gcp/Dockerfile @@ -1,10 +1,17 @@ -FROM python:3.7-slim-buster +# This Dockerfile assumes you used "make build" to convert the AdminUI to static files in frontend/adminui/dist/policyservice +FROM python:3.9-slim-buster -WORKDIR /opt - -COPY ./ /opt +# set environment variables +# PYTHONDONTWRITEBYTECODE - Prevents Python from writing pyc files to disc (equivalent to python -B option) +ENV PYTHONDONTWRITEBYTECODE 1 +# PYTHONUNBUFFERED - Prevents Python from buffering stdout and stderr (equivalent to python -u option) +ENV PYTHONUNBUFFERED 1 -RUN pip install -r requirements.txt +WORKDIR /opt +COPY ./app /opt +RUN pip install --no-cache-dir --upgrade -r /opt/requirements.txt -ENTRYPOINT python ./server.py +# Admin UI POC +#COPY ./frontend/adminui/dist/policyservice /opt/assets +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8080"] -- GitLab From 1ba526c00d3a0faf07eab83621b08dcc94078c4d Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Mon, 19 Sep 2022 10:59:28 -0400 Subject: [PATCH 04/10] moving to port 8080 --- deployment/service-policyservice.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deployment/service-policyservice.yaml b/deployment/service-policyservice.yaml index fe83fe1..712b79e 100644 --- a/deployment/service-policyservice.yaml +++ b/deployment/service-policyservice.yaml @@ -10,6 +10,6 @@ spec: ports: - protocol: TCP port: 80 - targetPort: 80 + targetPort: 8080 selector: - app: policy-service \ No newline at end of file + app: policy-service -- GitLab From ffcebb7e391654aa7535f29cf0a280c389143a70 Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Mon, 19 Sep 2022 14:54:38 -0400 Subject: [PATCH 05/10] Updating Dockerfile with exposing port --- app/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/app/Dockerfile b/app/Dockerfile index 85e4199..853de28 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -6,6 +6,7 @@ FROM python:3.9-slim-buster ENV PYTHONDONTWRITEBYTECODE 1 # PYTHONUNBUFFERED - Prevents Python from buffering stdout and stderr (equivalent to python -u option) ENV PYTHONUNBUFFERED 1 +EXPOSE 8080/tcp WORKDIR /opt COPY ./app /opt -- GitLab From f2bdef55cca1b69bc5e7a1e525da9b61fc3050bb Mon Sep 17 00:00:00 2001 From: Shane Hutchins Date: Mon, 19 Sep 2022 15:52:47 -0400 Subject: [PATCH 06/10] merge from master and moving to port 8080 --- devops/gcp/deploy/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devops/gcp/deploy/templates/deployment.yaml b/devops/gcp/deploy/templates/deployment.yaml index f6b5203..bc308bd 100644 --- a/devops/gcp/deploy/templates/deployment.yaml +++ b/devops/gcp/deploy/templates/deployment.yaml @@ -28,7 +28,7 @@ spec: allowPrivilegeEscalation: false runAsUser: 0 ports: - - containerPort: 80 + - containerPort: 8080 resources: requests: cpu: "{{ .Values.data.requestsCpu }}" -- GitLab From dec44d605f3bb5ce13c086bf275c8f0c797d08c1 Mon Sep 17 00:00:00 2001 From: Yauheni Rykhter Date: Tue, 20 Sep 2022 17:28:00 +0300 Subject: [PATCH 07/10] test image with gcloud+python3.10 --- devops/gcp/pipeline/override-stages.yml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index c2c1f57..8eeca01 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -234,13 +234,20 @@ osdu-gcp-dev2-bootstrap: - if: '$OSDU_GCP == "1" && $CI_COMMIT_TAG' when: on_success +#FIXME there is no python 3.9 image with gcloud osdu-gcp-test-python: tags: ["osdu-small"] - image: gcr.io/google.com/cloudsdktool/cloud-sdk + image: ubuntu:latest needs: ["osdu-gcp-bootstrap"] script: - - !reference [.common_test_config, script] - > + - apt-get update && apt-get install -y build-essential git libjpeg-dev curl python3 python3-pip apt-transport-https ca-certificates gnupg + - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list + - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - + - apt-get update && apt-get install google-cloud-cli + - echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > OSDU_GCP_INTEGRATION_TESTER.json + - gcloud auth activate-service-account --key-file OSDU_GCP_INTEGRATION_TESTER.json + - gcloud config set project $OSDU_GCP_PROJECT - export BEARER_TOKEN=`gcloud auth print-access-token` - pip install -r requirements.txt - pip install -r requirements_dev.txt @@ -298,6 +305,7 @@ osdu-gcp-anthos-test: when: never osdu-gcp-anthos-test-python: + image: python:3.9 rules: - if: "$CI_COMMIT_BRANCH =~ /^release/" when: never -- GitLab From 2e351f731bdf826c759810a43b635890fdd6e4fb Mon Sep 17 00:00:00 2001 From: Yauheni Rykhter Date: Tue, 20 Sep 2022 19:26:43 +0300 Subject: [PATCH 08/10] test image with gcloud+python3.9 --- devops/gcp/pipeline/override-stages.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index 8eeca01..c82f5d3 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -237,14 +237,21 @@ osdu-gcp-dev2-bootstrap: #FIXME there is no python 3.9 image with gcloud osdu-gcp-test-python: tags: ["osdu-small"] - image: ubuntu:latest + image: ubuntu:20.04 needs: ["osdu-gcp-bootstrap"] script: - > - - apt-get update && apt-get install -y build-essential git libjpeg-dev curl python3 python3-pip apt-transport-https ca-certificates gnupg + - apt-get update && apt-get install -y apt-transport-https ca-certificates gnupg curl - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - - apt-get update && apt-get install google-cloud-cli + - export DEBIAN_FRONTEND=noninteractive + - apt update + - apt -y install software-properties-common + - add-apt-repository -y ppa:deadsnakes/ppa + - apt -y install python3.9 + - python3.9 -m venv env + - source env/bin/activate - echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > OSDU_GCP_INTEGRATION_TESTER.json - gcloud auth activate-service-account --key-file OSDU_GCP_INTEGRATION_TESTER.json - gcloud config set project $OSDU_GCP_PROJECT @@ -254,7 +261,6 @@ osdu-gcp-test-python: - cd app - pip install -r requirements.txt - python3 -m pytest --token=$BEARER_TOKEN --service_url=$OSDU_GCP_POLICY_URL --data_partition=$DATA_PARTITION - osdu-gcp-dev2-test-python: tags: ["osdu-small"] image: gcr.io/google.com/cloudsdktool/cloud-sdk -- GitLab From 154a8916d24b384afd18183ecdea9c77e747290f Mon Sep 17 00:00:00 2001 From: Yauheni Rykhter Date: Tue, 20 Sep 2022 19:58:03 +0300 Subject: [PATCH 09/10] test image with gcloud+python3.9 --- devops/gcp/pipeline/override-stages.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index c82f5d3..712316f 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -246,10 +246,9 @@ osdu-gcp-test-python: - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - - apt-get update && apt-get install google-cloud-cli - export DEBIAN_FRONTEND=noninteractive - - apt update - apt -y install software-properties-common - add-apt-repository -y ppa:deadsnakes/ppa - - apt -y install python3.9 + - apt -y install python3.9 python3.9-venv - python3.9 -m venv env - source env/bin/activate - echo $OSDU_GCP_INTEGRATION_TESTER | base64 -d > OSDU_GCP_INTEGRATION_TESTER.json -- GitLab From 7417731e8637e118fd3e3a31e1bf954b60d85217 Mon Sep 17 00:00:00 2001 From: Yauheni Rykhter Date: Wed, 21 Sep 2022 13:18:46 +0300 Subject: [PATCH 10/10] add expose to image and add search.rego --- build/gcp/Dockerfile | 1 + devops/gcp/pipeline/override-stages.yml | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/build/gcp/Dockerfile b/build/gcp/Dockerfile index 566b9cc..9b12dcb 100644 --- a/build/gcp/Dockerfile +++ b/build/gcp/Dockerfile @@ -6,6 +6,7 @@ FROM python:3.9-slim-buster ENV PYTHONDONTWRITEBYTECODE 1 # PYTHONUNBUFFERED - Prevents Python from buffering stdout and stderr (equivalent to python -u option) ENV PYTHONUNBUFFERED 1 +EXPOSE 8080/tcp WORKDIR /opt COPY ./app /opt diff --git a/devops/gcp/pipeline/override-stages.yml b/devops/gcp/pipeline/override-stages.yml index 712316f..91879e8 100644 --- a/devops/gcp/pipeline/override-stages.yml +++ b/devops/gcp/pipeline/override-stages.yml @@ -156,7 +156,7 @@ osdu-gcp-bootstrap: - !reference [.common_test_config, script] - > - pip install -r requirements_bootstrap.txt - - tar -czf bundle.tar.gz ./deployment/default-policies/.manifest ./deployment/default-policies/dataauthz.rego ./deployment/default-policies/entitlements.rego ./deployment/default-policies/legal.rego + - tar -czf bundle.tar.gz ./deployment/default-policies/.manifest ./deployment/default-policies/dataauthz.rego ./deployment/default-policies/entitlements.rego ./deployment/default-policies/legal.rego ./deployment/default-policies/search.rego - mkdir --parents ./policies ; mv bundle.tar.gz $_ - python3 devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py - mv bundle-$DATA_PARTITION.tar.gz ./policies @@ -179,7 +179,7 @@ osdu-gcp-bootstrap-bundle: - !reference [.common_test_config, script] - > - pip install -r requirements_bootstrap.txt - - tar -czf bundle.tar.gz ./deployment/default-policies/.manifest ./deployment/default-policies/dataauthz.rego ./deployment/default-policies/entitlements.rego ./deployment/default-policies/legal.rego + - tar -czf bundle.tar.gz ./deployment/default-policies/.manifest ./deployment/default-policies/dataauthz.rego ./deployment/default-policies/entitlements.rego ./deployment/default-policies/legal.rego ./deployment/default-policies/search.rego - mkdir --parents ./policies ; mv bundle.tar.gz $_ - python3 devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py - mv bundle-$DATA_PARTITION.tar.gz ./policies @@ -202,7 +202,7 @@ osdu-gcp-dev2-bootstrap-bundle: - !reference [.common_test_config, script] - > - pip install -r requirements_bootstrap.txt - - tar -czf bundle.tar.gz ./deployment/default-policies/.manifest ./deployment/default-policies/dataauthz.rego ./deployment/default-policies/entitlements.rego ./deployment/default-policies/legal.rego + - tar -czf bundle.tar.gz ./deployment/default-policies/.manifest ./deployment/default-policies/dataauthz.rego ./deployment/default-policies/entitlements.rego ./deployment/default-policies/legal.rego ./deployment/default-policies/search.rego - mkdir --parents ./policies ; mv bundle.tar.gz $_ - python3 devops/gcp/bootstrap-osdu-module/DataPartitionBundles.py - mv bundle-$DATA_PARTITION.tar.gz ./policies @@ -244,7 +244,7 @@ osdu-gcp-test-python: - apt-get update && apt-get install -y apt-transport-https ca-certificates gnupg curl - echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - - - apt-get update && apt-get install google-cloud-cli + - apt-get update && apt-get -y install google-cloud-cli - export DEBIAN_FRONTEND=noninteractive - apt -y install software-properties-common - add-apt-repository -y ppa:deadsnakes/ppa @@ -260,6 +260,7 @@ osdu-gcp-test-python: - cd app - pip install -r requirements.txt - python3 -m pytest --token=$BEARER_TOKEN --service_url=$OSDU_GCP_POLICY_URL --data_partition=$DATA_PARTITION + osdu-gcp-dev2-test-python: tags: ["osdu-small"] image: gcr.io/google.com/cloudsdktool/cloud-sdk -- GitLab