Commit 894b047f authored by Riabokon Stanislav(EPAM)[GCP]'s avatar Riabokon Stanislav(EPAM)[GCP]
Browse files

Finish development of Entitlements V2 GCP

parent 58465585
**/.idea
**/target
**/build
**/.gradle
**/out
**/*.iml
......
......@@ -13,6 +13,13 @@ variables:
IBM_INT_TEST_SUBDIR: testing/entitlements-v2-test-ibm
OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
OSDU_GCP_SERVICE: entitlements-v2
OSDU_GCP_VENDOR: jdbc
OSDU_GCP_HELM_NAMESPACE: default
OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.domain=$DOMAIN --set data.spring_datasource_url=jdbc:postgresql://127.0.0.1:5432/entitlements --set data.spring_datasource_password=$OSDU_GCP_SPRING_DATASOURCE_PASSWORD --set data.spring_datasource_username=postgres --set data.partition_api=$MY_TENANT --set data.google_audiences=$GOOGLE_AUDIENCE --set data.log_level=INFO --set data.partition_api=$OSDU_GCP_PARTITION_API"
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=workload-identity-entitlements --set data.sql_connection_string=nice-etching-277309:us-central1:entitlements-v2"
OSDU_GCP_HELM_CONFIG_SERVICE: entitlements-config
OSDU_GCP_HELM_DEPLOYMENT_SERVICE: entitlements-deploy
include:
......@@ -43,7 +50,13 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/ibm.yml"
#Include osdu-gcp-global.yml at k8s common pipeline
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/osdu-gcp-gke.yml"
#Include osdu-gcp-global.yml at k8s common pipeline
- project: "osdu/platform/ci-cd-pipelines"
file: 'cloud-providers/osdu-gcp-global.yml'
osdu-gcp-deploy-deployment:
variables:
OSDU_GCP_SERVICE: entitlements
......@@ -152,7 +152,7 @@ The following software have components provided under the terms of this license:
- Jackson module: Afterburner (from https://github.com/FasterXML/jackson-modules-base)
- Jackson module: JAXB-annotations (from http://github.com/FasterXML/jackson-module-jaxb-annotations)
- Jackson-annotations (from http://github.com/FasterXML/jackson)
- Jackson-core (from http://wiki.fasterxml.com/JacksonHome)
- Jackson-core (from https://github.com/FasterXML/jackson-core)
- Jackson-dataformat-XML (from http://wiki.fasterxml.com/JacksonExtensionXmlDataBinding)
- Jackson-dataformat-YAML (from https://github.com/FasterXML/jackson-dataformats-text)
- Jackson-datatype-Joda (from http://wiki.fasterxml.com/JacksonModuleJoda)
......@@ -234,18 +234,6 @@ The following software have components provided under the terms of this license:
- RxJava (from https://github.com/ReactiveX/RxJava)
- RxJava (from https://github.com/ReactiveX/RxJava)
- SnakeYAML (from http://code.google.com/p/snakeyaml/)
- Spring Boot (from http://projects.spring.io/spring-boot/)
- Spring Boot Actuator (from http://projects.spring.io/spring-boot/)
- Spring Boot Actuator AutoConfigure (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-actuator-autoconfigure)
- Spring Boot AutoConfigure (from http://projects.spring.io/spring-boot/)
- Spring Boot Dependencies (from http://projects.spring.io/spring-boot/)
- Spring Boot Json Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-json)
- Spring Boot Log4J2 Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Logging Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Security Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Test Auto-Configure (from http://projects.spring.io/spring-boot/)
- Spring Boot Test Starter (from http://projects.spring.io/spring-boot/)
- Spring Boot Undertow Starter (from https://projects.spring.io/spring-boot/#/spring-boot-parent/spring-boot-starters/spring-boot-starter-undertow)
- Spring Commons Logging Bridge (from https://github.com/spring-projects/spring-framework)
- Spring Data Core (from https://repo1.maven.org/maven2/org/springframework/data/spring-data-commons)
......@@ -300,7 +288,7 @@ The following software have components provided under the terms of this license:
- io.grpc:grpc-stub (from https://github.com/grpc/grpc-java)
- ion-java (from https://github.com/amznlabs/ion-java/)
- ion-java (from https://github.com/amznlabs/ion-java/)
- jackson-databind (from http://wiki.fasterxml.com/JacksonHome)
- jackson-databind (from http://github.com/FasterXML/jackson)
- java-cloudant (from https://cloudant.com)
- java-cloudant (from https://cloudant.com)
- javatuples (from http://www.javatuples.org)
......@@ -331,11 +319,23 @@ The following software have components provided under the terms of this license:
- resilience4j (from https://github.com/resilience4j/resilience4j)
- resilience4j (from https://github.com/resilience4j/resilience4j)
- rxjava (from https://github.com/ReactiveX/RxJava)
- spring-boot (from https://spring.io/projects/spring-boot)
- spring-boot-actuator (from https://spring.io/projects/spring-boot)
- spring-boot-actuator-autoconfigure (from https://spring.io/projects/spring-boot)
- spring-boot-autoconfigure (from https://spring.io/projects/spring-boot)
- spring-boot-dependencies (from https://spring.io/projects/spring-boot)
- spring-boot-starter (from https://spring.io/projects/spring-boot)
- spring-boot-starter-actuator (from https://spring.io/projects/spring-boot)
- spring-boot-starter-json (from https://spring.io/projects/spring-boot)
- spring-boot-starter-log4j2 (from https://spring.io/projects/spring-boot)
- spring-boot-starter-logging (from https://spring.io/projects/spring-boot)
- spring-boot-starter-security (from https://spring.io/projects/spring-boot)
- spring-boot-starter-test (from https://spring.io/projects/spring-boot)
- spring-boot-starter-tomcat (from https://spring.io/projects/spring-boot)
- spring-boot-starter-validation (from https://spring.io/projects/spring-boot)
- spring-boot-starter-web (from https://spring.io/projects/spring-boot)
- spring-boot-test (from https://spring.io/projects/spring-boot)
- spring-boot-test-autoconfigure (from https://spring.io/projects/spring-boot)
- spring-security-config (from http://spring.io/spring-security)
- spring-security-core (from https://spring.io/spring-security)
- spring-security-oauth2-client (from http://spring.io/spring-security)
......
How to run integration tests in local environment for azure implementation,
please refer to: testing/entitlements-v2-test-azure/README.md
### Integration tests
Instructions for running the Azure integration tests in local environment can be found [here][Azure documentation]
Instructions for running the JDBC integration tests can be found [here][JDBC documentation].
[Azure documentation]: testing/entitlements-v2-test-azure/README.md
[JDBC documentation]: provider/entitlements-v2-jdbc/README.md
......@@ -3,6 +3,8 @@ kind: Secret
metadata:
labels:
app: "{{ .Values.conf.app_name }}"
annotations:
rollme: {{ randAlphaNum 5 | quote }}
name: "{{ .Values.conf.secret_name }}"
namespace: "{{ .Release.Namespace }}"
type: Opaque
......
......@@ -3,12 +3,12 @@ kind: ConfigMap
metadata:
labels:
app: "{{ .Values.conf.app_name }}"
annotations:
rollme: {{ randAlphaNum 5 | quote }}
name: "{{ .Values.conf.configmap }}"
namespace: "{{ .Release.Namespace }}"
data:
DOMAIN: "{{ .Values.data.domain }}"
REDIS_GROUP_HOST: "{{ .Values.data.redis_group_host }}"
REDIS_GROUP_PORT: "{{ .Values.data.redis_group_port }}"
SPRING_DATASOURCE_URL: "{{ .Values.data.spring_datasource_url }}"
GOOGLE_AUDIENCES: "{{ .Values.data.google_audiences }}"
SPRING_DATASOURCE_USERNAME: "{{ .Values.data.spring_datasource_username }}"
......
data:
domain: ""
google_audiences: ""
redis_group_host: ""
redis_group_port: 9423
spring_datasource_url: ""
spring_datasource_username: ""
spring_datasource_password: ""
......@@ -13,5 +11,5 @@ data:
conf:
configmap: "entitlements-config"
app_name: "entitlements-sql"
app_name: "entitlements"
secret_name: "entitlements-secret"
......@@ -3,6 +3,8 @@ kind: Deployment
metadata:
name: "{{ .Values.conf.app_name }}"
namespace: "{{ .Release.Namespace }}"
annotations:
rollme: {{ randAlphaNum 5 | quote }}
spec:
replicas: 1
selector:
......
......@@ -6,5 +6,5 @@ data:
conf:
configmap: "entitlements-config"
app_name: "entitlements-sql"
app_name: "entitlements"
secret_name: "entitlements-secret"
# JDBC Entitlements
## Database structure
![Jdbc diagram](jdbc.png)
## Entitlements tables
### Group
Group is a structure that provides specific access to its members.
#### id
This is the unique identifier of a group in the table.
**Value type:** `bigint`
**Properties:** NOT NULL, GENERATED, IDENTITY, **PRIMARY KEY**
#### name
The group name. Usually, it has the following format: `{groupType}.{resourceName}.{permission}`
Where:
* `groupType` - type of the group (data, service, users, etc.);
* `resourceName` - name of a service group belongs to;
* `permission` - type of data access (viewers, owners).
**Value type:** `varchar`
**Properties:** UNIQUE
#### description
The group description in free format.
**Value type:** `text`
#### email
The group email. It is built by this format: `{name}@{data-partition-id}.{domain}.com`
**Value type:** `varchar`
**Properties:** UNIQUE
#### partition_id
The partition id of a group.
**Value type:** `varchar`
___
### Member
#### id
This is the unique identifier of a member in the table.
**Value type:** `bigint`
**Properties:** NOT NULL, GENERATED, IDENTITY, **PRIMARY KEY**
#### email
The member email. It is typically in this format: `member@domain.com`
**Value type:** `varchar`
**Properties:** UNIQUE
#### partition_id
The partition id of a member.
**Value type:** `varchar`
___
### Member to Group
Group and member connected by `Many-to-Many` relationship through this table. The table also
contains the role of the member (`OWNER` or `MEMBER`).
#### group_id
This is the unique identifier of a group (the foreign key of `id` from the `group` table).
**PRIMARY KEY** - (`group_id`, `member_id`)
**Value type:** `bigint`
**Properties:** NOT NULL, the part of **PRIMARY KEY**
#### member_id
This is the unique identifier of a member (the foreign key of `id` from the `member` table).
**Value type:** `bigint`
**Properties:** NOT NULL, the part of **PRIMARY KEY**
#### role
The role of a member (`OWNER` or `MEMBER`).
**Value type:** `varchar`
___
### Embedded group
Embedded group implemented by `Many-to-Many` relationship through this table.
#### parent_id
This is the unique identifier of a parent group (the foreign key of `id` from the `group` table).
**Value type:** `bigint`
**Properties:** NOT NULL, the part of **PRIMARY KEY**
#### child_id
This is the unique identifier of a child group (the foreign key of `id` from the `group` table).
**Value type:** `bigint`
**Properties:** NOT NULL, the part of **PRIMARY KEY**
### App Id
App id is one-to-many table for groups to support V2 functionality.
#### id
This is the unique identifier of app id relation in the table.
#### group_id
This is the unique identifier of a group (the foreign key of `id` from the `group` table).
**Value type:** `bigint`
**Properties:** NOT NULL
#### role
The app id of the particular group.
**Value type:** `varchar`
\ No newline at end of file
......@@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.exc.UnrecognizedPropertyException;
import org.opengroup.osdu.core.common.logging.JaxRsDpsLog;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.partition.PartitionException;
import org.opengroup.osdu.entitlements.v2.validation.PartitionHeaderValidationService;
import org.springframework.beans.factory.BeanCreationException;
import org.springframework.beans.factory.annotation.Autowired;
......@@ -32,6 +33,10 @@ public class SpringExceptionMapper extends ResponseEntityExceptionHandler {
@ExceptionHandler(AppException.class)
protected ResponseEntity<Object> handleAppException(AppException e) {
if (e.getOriginalException() instanceof PartitionException) {
e.getError().setCode(HttpStatus.UNAUTHORIZED.value());
}
return this.getErrorResponse(e);
}
......
# entitlements-v2-jdbc
entitlements-v2-jdbc is a [Spring Boot](https://spring.io/projects/spring-boot) service which hosts CRUD APIs that enable management of user entitlements.
Data kept in GCP instance of Postgres database.
### Database structure
The database used in this implementation is PostgreSQL 13.0. The database structure and additional
info can be found [here][JDBC documentation].
### Requirements
In order to run this service from a local machine to Cloud Run, you need the following:
- [Maven 3.6.0+](https://maven.apache.org/download.cgi)
- [AdoptOpenJDK8](https://adoptopenjdk.net/)
- [Google Cloud SDK](https://cloud.google.com/sdk/)
- [Docker](https://docs.docker.com/engine/install/)
### General Tips
**Environment Variable Management**
The following tools make environment variable configuration simpler
- [direnv](https://direnv.net/) - for a shell/terminal environment
- [EnvFile](https://plugins.jetbrains.com/plugin/7861-envfile) - for [Intellij IDEA](https://www.jetbrains.com/idea/)
**Lombok**
This project uses [Lombok](https://projectlombok.org/) for code generation. You may need to configure your IDE to take advantage of this tool.
- [Intellij configuration](https://projectlombok.org/setup/intellij)
- [VSCode configuration](https://projectlombok.org/setup/vscode)
### Environment Variables
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `SPRING_DATASOURCE_URL` | ex `jdbc:postgresql://localhost:5432/entitlements` | The JDBC-valid connection string for database | yes | https://console.cloud.google.com/ |
| `SPRING_DATASOURCE_USERNAME` | ex `postgres` | The username of database user | yes | - |
| `SPRING_DATASOURCE_PASSWORD` | ex `********` | The password of database user | yes | - |
| `SPRING_PROFILES_ACTIVE` | ex `dev` | Spring profile to be active | no | - |
| `LOG_PREFIX` | `entitlements-v2` | Logging prefix | no | - |
| `LOG_LEVEL` | `INFO` | Logging level | no | - |
| `DOMAIN` | ex `opendes.com` | The name of the domain for which the service runs | no | -- |
| `PARTITION_API` | ex `http://localhost:8080/api/partition/v1` | Partition service endpoint | no | - |
| `server_port` | ex `8080` | Port of the server | no | -- |
| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `GOOGLE_APPLICATION_CREDENTIALS` | `********` | Need this only if running locally, this service acc must have token sign access | yes | -- |
### Build and run the application
After configuring your environment as specified above, you can follow these steps to build and run the application. These steps should be invoked from the *repository root.*
```bash
# build + test + install core service code
$ ./mvnw clean install
# run service
#
# Note: this assumes that the environment variables for running the service as outlined
# above are already exported in your environment.
$ java -jar $(find provider/entitlements-v2-jdbc/target/ -name '*-spring-boot.jar')
# Alternately you can run using the Maven Task
$ ./mvnw spring-boot:run -pl provider/entitlements-v2-jdbc
```
### Test the application
### Integration Tests
In order to run integration tests, you need to have the following environment variables defined:
**Required to run integration tests**
| Name | Value | Description | Sensitive? | Source |
| --- | --- | --- | --- | --- |
| `ENTITLEMENT_V2_URL` | ex `http://localhost:8080/api/entitlements/v2/` | The host where the service is running | no | -- |
| `DOMAIN` | ex `contoso.com` | Must match the value of `service_domain_name` above | no | -- |
| `TENANT_NAME` | ex `opendes` | OSDU tenant used for testing | no | -- |
| `INTEGRATION_TESTER` | `********` | System identity to assume for API calls. Note: This user must have entitlements already configured | yes | -- |
| `INTEGRATION_TEST_AUDIENCE` | `********` | Client Id for `$INTEGRATION_TESTER` | yes | -- |
| `GOOGLE_APPLICATION_CREDENTIALS` | `********` | System identity to provide access for cleaning up groups created during test | yes | -- |
**Entitlements configuration for integration accounts**
| INTEGRATION_TESTER |
| --- |
| users<br/>service.entitlements.user<br/>service.entitlements.admin |
#### Using Cloud Infrastructure
1. Run Entitlements V2 service from JDBC provider (assumed that all the required environment variables specified for using Cloud Infrastructure).
2. Define environment variables for integration tests (e.g. maven options).
3. Run integration tests:
```bash
# build + install integration test core
$ ./mvnw compile -f testing/entitlements-v2-test-core
# build + run JDBC integration tests.
$ ./mvnw test -f testing/entitlements-v2-test-jdbc
```
## Debugging
Jet Brains - the authors of Intellij IDEA, have written an [excellent guide](https://www.jetbrains.com/help/idea/debugging-your-first-java-application.html) on how to debug java programs.
## License
Copyright © EPAM Systems
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0)
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
[JDBC Documentation]: ../../docs/JDBC.md
\ No newline at end of file
# THIS ASAP manual BUILD jar ent v2
# Use the official AdoptOpenJDK for a base image.
# https://hub.docker.com/_/openjdk
FROM openjdk:8-slim
WORKDIR /app
ARG PROVIDER_NAME
ENV PROVIDER_NAME $PROVIDER_NAME
ARG PORT
ENV PORT $PORT
## Copy the jar to the production image from the builder stage.
COPY provider/entitlements-v2-${PROVIDER_NAME}/target/entitlements-v2-${PROVIDER_NAME}-*-spring-boot.jar entitlements-${PROVIDER_NAME}.jar
# Run the web service on container startup.
CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=${PORT} -jar entitlements-${PROVIDER_NAME}.jar
# Copyright 2020 Google LLC
# Copyright 2017-2019, Schlumberger
# Copyright 2020 EPAM
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
steps:
- name: 'gcr.io/cloud-builders/docker'
args: [
'build',
'--build-arg', 'PROVIDER_NAME=${_PROVIDER_NAME}',
'--build-arg', 'PORT=${_PORT}',
'-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:${_SHORT_SHA}',
'-t', 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}:latest',
'-f', 'provider/${_GCP_SERVICE}-${_PROVIDER_NAME}/cloudbuild/Dockerfile.cloudbuild',
'.'
]
images:
- 'gcr.io/$PROJECT_ID/${_APPLICATION_NAME}/${_GCP_SERVICE}-${_PROVIDER_NAME}'
DROP TABLE IF EXISTS public.member_to_group;
DROP TABLE IF EXISTS public."member";
DROP TABLE IF EXISTS public.embedded_group;
DROP TABLE IF EXISTS public.app_id;
DROP TABLE IF EXISTS public."group";
CREATE TABLE public."group"
(
id bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
name character varying COLLATE pg_catalog."default",
description text COLLATE pg_catalog."default",
email character varying COLLATE pg_catalog."default",
partition_id character varying COLLATE pg_catalog."default",
CONSTRAINT group_pkey PRIMARY KEY (id),
CONSTRAINT group_email_key UNIQUE (email),
CONSTRAINT group_name_key UNIQUE (name)
)
TABLESPACE pg_default;
ALTER TABLE public."group"
OWNER to postgres;
CREATE TABLE public.app_id
(
id bigint NOT NULL GENERATED ALWAYS AS IDENTITY,
group_id bigint,
app_id character varying,
PRIMARY KEY (id),
CONSTRAINT app_id_group_fk FOREIGN KEY (group_id)
REFERENCES public."group" (id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
NOT VALID
);
ALTER TABLE public.app_id
OWNER to postgres;
CREATE TABLE public.member
(
id bigint NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 9223372036854775807 CACHE 1 ),
email character varying COLLATE pg_catalog."default",
partition_id character varying COLLATE pg_catalog."default",
CONSTRAINT member_pkey PRIMARY KEY (id),
CONSTRAINT member_email_key UNIQUE (email)
)
TABLESPACE pg_default;
ALTER TABLE public.member
OWNER to postgres;
CREATE TABLE public.member_to_group
(
group_id bigint NOT NULL,
member_id bigint NOT NULL,
role character varying COLLATE pg_catalog."default",
CONSTRAINT member_to_group_pkey PRIMARY KEY (group_id, member_id),
CONSTRAINT group_as_member_holder_fk FOREIGN KEY (group_id)
REFERENCES public."group" (id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
NOT VALID,
CONSTRAINT user_as_member_fk FOREIGN KEY (member_id)
REFERENCES public.member (id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
NOT VALID
)
TABLESPACE pg_default;
ALTER TABLE public.member_to_group
OWNER to postgres;
CREATE TABLE public.embedded_group
(
parent_id bigint NOT NULL,
child_id bigint NOT NULL,
CONSTRAINT embedded_group_pk PRIMARY KEY (parent_id, child_id),
CONSTRAINT group_as_child_fk FOREIGN KEY (child_id)
REFERENCES public."group" (id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
NOT VALID,
CONSTRAINT group_as_parent_fk FOREIGN KEY (parent_id)
REFERENCES public."group" (id) MATCH SIMPLE
ON UPDATE NO ACTION
ON DELETE NO ACTION
NOT VALID
)
TABLESPACE pg_default;
ALTER TABLE public.embedded_group
OWNER to postgres;
\ No newline at end of file