Commit 844a7018 authored by Harshit Saxena's avatar Harshit Saxena
Browse files

Merge branch 'master' of...

Merge branch 'master' of https://community.opengroup.org/osdu/platform/system/dataset into file_collection
parents eee0db35 9b3224c7
Pipeline #87324 failed with stages
in 7 seconds
......@@ -9,7 +9,7 @@ variables:
IBM_BUILD_SUBDIR: provider/dataset-ibm
IBM_INT_TEST_SUBDIR: testing/dataset-test-ibm
# --- osdu-gcp specific variables ---
# --- osdu-gcp specific variables ---
OSDU_GCP_SERVICE: dataset
OSDU_GCP_VENDOR: gcp
OSDU_GCP_APPLICATION_NAME: dataset
......@@ -18,7 +18,7 @@ variables:
OSDU_GCP_TEST_SUBDIR: testing/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.google_audiences=$GOOGLE_AUDIENCE --set data.redis_group_host=$REDIS_GROUP_HOST --set data.file_dms_bucket=$OSDU_GCP_FILE_DMS_BUCKET --set data.expiration_days=$OSDU_GCP_EXPIRATION_DAYS"
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=workload-identity-dataset"
OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s"
OSDU_GCP_HELM_CONFIG_SERVICE: dataset-config
OSDU_GCP_HELM_DEPLOYMENT_SERVICE: dataset-deploy
......@@ -37,14 +37,14 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/fossa-maven.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/aws-global.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/aws-global.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/aws-maven.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/aws-maven.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/azure.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/azure.yml"
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/ibm.yml"
......@@ -52,9 +52,8 @@ include:
- project: "osdu/platform/ci-cd-pipelines"
file: "scanners/gitlab-ultimate.yml"
- project: 'osdu/platform/ci-cd-pipelines'
file: 'cloud-providers/osdu-gcp-gke.yml'
- project: "osdu/platform/ci-cd-pipelines"
file: "cloud-providers/osdu-gcp-gke.yml"
# disable the eslint scanner
# I think this is being generated from the presence of an HTML file, but there
......
......@@ -34,11 +34,8 @@
<java.version>1.8</java.version>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<springfox-version>2.7.0</springfox-version>
<tomcat_embed_core_version>9.0.54</tomcat_embed_core_version>
<!--
<springfox-version>2.9.2</springfox-version>
-->
<log4j2.version>2.17.0</log4j2.version>
</properties>
<dependencies>
......@@ -79,8 +76,35 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-tomcat</artifactId>
</exclusion>
</exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
<version>${log4j2.version}</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-security</artifactId>
......@@ -148,18 +172,6 @@
<version>3.0.0</version>
<scope>test</scope>
</dependency>
<!-- swagger dependencies -->
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger2</artifactId>
<version>${springfox-version}</version>
</dependency>
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-swagger-ui</artifactId>
<version>${springfox-version}</version>
</dependency>
</dependencies>
<build>
......
package org.opengroup.osdu.dataset.swagger;
import io.swagger.v3.oas.models.OpenAPI;
import io.swagger.v3.oas.models.servers.Server;
import org.springframework.core.Ordered;
import org.springframework.core.annotation.Order;
import org.springframework.stereotype.Component;
import springfox.documentation.oas.web.OpenApiTransformationContext;
import springfox.documentation.oas.web.WebMvcOpenApiTransformationFilter;
import springfox.documentation.spi.DocumentationType;
import javax.servlet.http.HttpServletRequest;
@Component
@Order(Ordered.LOWEST_PRECEDENCE)
public class SpringfoxSwaggerHostResolver implements WebMvcOpenApiTransformationFilter {
@Override
public boolean supports(DocumentationType delimiter) {
return delimiter == DocumentationType.OAS_30;
}
@Override
public OpenAPI transform(OpenApiTransformationContext<HttpServletRequest> context) {
OpenAPI swagger = context.getSpecification();
Server server = swagger.getServers().get(0);
if (server.getUrl().contains(":443")) {
// via the gateway
server.setUrl(server.getUrl().replace(":443",""));
}
return swagger;
}
}
\ No newline at end of file
......@@ -19,25 +19,28 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Profile;
import springfox.documentation.builders.ParameterBuilder;
import springfox.documentation.builders.PathSelectors;
import springfox.documentation.builders.RequestHandlerSelectors;
import springfox.documentation.schema.ModelRef;
import springfox.documentation.builders.RequestParameterBuilder;
import springfox.documentation.oas.annotations.EnableOpenApi;
import springfox.documentation.service.ApiKey;
import springfox.documentation.service.AuthorizationScope;
import springfox.documentation.service.ParameterType;
import springfox.documentation.service.RequestParameter;
import springfox.documentation.service.SecurityReference;
import springfox.documentation.spi.DocumentationType;
import springfox.documentation.spi.service.contexts.SecurityContext;
import springfox.documentation.spring.web.plugins.Docket;
import springfox.documentation.swagger2.annotations.EnableSwagger2;
import springfox.documentation.service.Parameter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@Configuration
@EnableSwagger2
@EnableOpenApi
@Profile("!noswagger")
public class SwaggerDocumentationConfig {
public static final String AUTHORIZATION_HEADER = "Authorization";
......@@ -45,26 +48,23 @@ public class SwaggerDocumentationConfig {
@Bean
public Docket api() {
ParameterBuilder builder = new ParameterBuilder();
List<Parameter> parameters = new ArrayList<>();
RequestParameterBuilder builder = new RequestParameterBuilder();
List<RequestParameter> parameters = new ArrayList<>();
builder.name(DpsHeaders.DATA_PARTITION_ID)
.description("tenant")
.defaultValue("common")
.modelRef(new ModelRef("string"))
.parameterType("header")
.in(ParameterType.HEADER)
.required(true)
.build();
parameters.add(builder.build());
builder.name("frame-of-reference")
.description("reference")
.defaultValue("none")
.modelRef(new ModelRef("string"))
.parameterType("header")
.in(ParameterType.HEADER)
.required(true)
.build();
parameters.add(builder.build());
return new Docket(DocumentationType.SWAGGER_2)
.globalOperationParameters(parameters)
return new Docket(DocumentationType.OAS_30)
.globalRequestParameters(parameters)
.select()
.apis(RequestHandlerSelectors.basePackage("org.opengroup.osdu.dataset.api"))
.build()
......@@ -73,13 +73,13 @@ public class SwaggerDocumentationConfig {
}
private ApiKey apiKey() {
return new ApiKey("JWT", AUTHORIZATION_HEADER, "header");
return new ApiKey(AUTHORIZATION_HEADER, AUTHORIZATION_HEADER, "header");
}
private SecurityContext securityContext() {
return SecurityContext.builder()
.securityReferences(defaultAuth())
.forPaths(PathSelectors.regex(DEFAULT_INCLUDE_PATTERN))
.operationSelector(o -> PathSelectors.regex(DEFAULT_INCLUDE_PATTERN).test(o.requestMappingPattern()))
.build();
}
......@@ -89,6 +89,6 @@ public class SwaggerDocumentationConfig {
AuthorizationScope[] authorizationScopes =
new AuthorizationScope[]{authorizationScope};
return Collections.singletonList(
new SecurityReference("JWT", authorizationScopes));
new SecurityReference(AUTHORIZATION_HEADER, authorizationScopes));
}
}
......@@ -26,12 +26,14 @@
<maven.compiler.source>1.8</maven.compiler.source>
<docker.image.prefix>opendes</docker.image.prefix>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<os-core-common.version>0.12.0</os-core-common.version>
<os-core-common.version>0.13.0-rc4</os-core-common.version>
<jackson.version>2.11.2</jackson.version>
<netty.version>4.1.51.Final</netty.version>
<snakeyaml.version>1.26</snakeyaml.version>
<commons-codec.version>1.14</commons-codec.version>
<nimbusds.version>7.9</nimbusds.version>
<log4j2.version>2.17.0</log4j2.version>
<springfox.version>3.0.0</springfox.version>
</properties>
<licenses>
......@@ -53,6 +55,21 @@
<type>pom</type>
<scope>import</scope>
</dependency> -->
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-to-slf4j</artifactId>
<version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
<version>${log4j2.version}</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>${os-core-common.version}</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
......@@ -85,11 +102,6 @@
<artifactId>snakeyaml</artifactId>
<version>${snakeyaml.version}</version>
</dependency> -->
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>os-core-common</artifactId>
<version>${os-core-common.version}</version>
</dependency>
</dependencies>
</dependencyManagement>
......@@ -100,6 +112,13 @@
<version>1.18.8</version>
<scope>provided</scope>
</dependency>
<!-- swagger dependency -->
<dependency>
<groupId>io.springfox</groupId>
<artifactId>springfox-boot-starter</artifactId>
<version>${springfox.version}</version>
</dependency>
</dependencies>
<profiles>
......
......@@ -58,7 +58,7 @@
<dependency>
<groupId>org.opengroup.osdu.core.aws</groupId>
<artifactId>os-core-lib-aws</artifactId>
<version>0.12.0</version>
<version>0.13.0-rc3</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
......
......@@ -37,8 +37,8 @@
<java.version>1.8</java.version>
<maven.compiler.target>1.8</maven.compiler.target>
<maven.compiler.source>1.8</maven.compiler.source>
<osdu.corelibazure.version>0.12.0</osdu.corelibazure.version>
<osdu.oscorecommon-version>0.12.0</osdu.oscorecommon-version>
<osdu.corelibazure.version>0.13.0-rc6</osdu.corelibazure.version>
<osdu.oscorecommon-version>0.13.0-rc4</osdu.oscorecommon-version>
<nimbus-jose-jwt.version>7.9</nimbus-jose-jwt.version>
</properties>
......
......@@ -42,11 +42,13 @@ public class AADSecurityConfig extends WebSecurityConfigurerAdapter {
.authorizeRequests()
.antMatchers("/", "/index.html",
"/v2/api-docs",
"/v3/api-docs",
"/configuration/ui",
"/swagger-resources/**",
"/configuration/security",
"/swagger",
"/swagger-ui.html",
"/swagger-ui/**",
"/info",
"/webjars/**").permitAll()
.anyRequest().authenticated()
......
......@@ -4,30 +4,210 @@
These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
### Prerequisites
Pre-requisites
* GCloud SDK with java (latest version)
* JDK 8
* Lombok 1.16 or later
* Maven
### Installation
In order to run the service locally or remotely, you will need to have the following environment variables defined.
# Features of implementation
This is a universal solution created using EPAM OSM and OBM mappers technology.
It allows you to work with various implementations of KV stores and Blob stores.
## Limitations of the current version
In the current version, the mappers are equipped with several drivers to the stores:
- OSM (mapper for KV-data): Google Datastore; Postgres
- OBM (mapper to Blob stores): Google Cloud Storage (GCS); MinIO
## Extensibility
To use any other store or message broker, implement a driver for it. With an extensible set of drivers, the solution is unrestrictedly universal and portable without modification to the main code.
Mappers support "multitenancy" with flexibility in how it is implemented.
They switch between datasources of different tenants due to the work of a bunch of classes that implement the following interfaces:
- Destination - takes a description of the current context, e.g., "data-partition-id = opendes"
- DestinationResolver – accepts Destination, finds the resource, connects, and returns Resolution
- DestinationResolution – contains a ready-made connection, the mapper uses it to get to data
## Mapper tuning mechanisms
This service uses specific implementations of DestinationResolvers based on the tenant information provided by the OSDU Partition service.
- for Google Datastore: osm/config/DsTenantDestinationResolver.java
- for Postgres: osm/config/PgTenantDestinationResolver.java
#### Their algorithms are as follows:
- incoming Destination carries data-partition-id
- resolver accesses the Partition service and gets PartitionInfo
- from PartitionInfo resolver retrieves properties for the connection: URL, username, password etc.
- resolver creates a data source, connects to the resource, remembers the datasource
- resolver gives the datasource to the mapper in the Resolution object
- Google Cloud resolvers do not receive special properties from the Partition service for connection,
because the location of the resources is unambiguously known - they are in the GCP project.
And credentials are also not needed - access to data is made on behalf of the Google Identity SA
under which the service itself is launched. Therefore, resolver takes only
the value of the **projectId** property from PartitionInfo and uses it to connect to a resource
in the corresponding GCP project.
# Configuration
## Service Configuration
Define the following environment variables.
Most of them are common to all hosting environments, but there are properties that are only necessary when running in Google Cloud.
#### Common properties for all environments
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `GCP_SCHEMA_API` | ex `https://os-schema-jvmvia5dea-uc.a.run.app/api/schema-service/v1` | Schema API endpoint | no | output of infrastructure deployment |
| `GCP_STORAGE_API` | ex `https://os-storage-jvmvia5dea-uc.a.run.app/api/storage/v2` | Storage API endpoint | no | output of infrastructure deployment |
| `SCHEMA_API` | ex `https://os-schema-jvmvia5dea-uc.a.run.app/api/schema-service/v1` | Schema API endpoint | no | output of infrastructure deployment |
| `STORAGE_API` | ex `https://os-storage-jvmvia5dea-uc.a.run.app/api/storage/v2` | Storage API endpoint | no | output of infrastructure deployment |
| `AUTHORIZE_API` | ex `https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
| `FILE_DMS_BUCKET` | ex `file-dms-bucket` | File bucket name postfix (full name represent by project-id + partition-id + GCP_FILE_DMS_BUCKET ex `osdu-cicd-epam-opendes-file-dms-bucket`) | no | output of infrastructure deployment |
| `EXPIRATION_DAYS` | ex `1` | expiration for signed urls & connection strings | no | |
| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
| `REDIS_GROUP_PORT` | ex `1111` | Redis port | no | https://console.cloud.google.com/memorystore/redis/instances |
| `osdu.dataset.config.useRestDms` | `true` OR `false` | Allows to configure *DMS REST APIs usage* | no | - |
| `DMS_API_BASE` | ex `http://localhost:8081/api/file/v2/files` | *Only for local usage.* Allows to override DMS service base url value from Datastore. | no | - |
#### For Mappers, to activate drivers
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `OSMDRIVER` | `postgres` OR `datastore` | Osm driver mode that defines which KV storage will be used | no | - |
| `OBMDRIVER` | `gcs` OR `minio` | Obm driver mode that defines which object storage will be used | no | - |
#### For Google Cloud only
| name | value | description | sensitive? | source |
| --- | --- | --- | --- | --- |
| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
| `osdu.dataset.config.useRestDms` | `true` OR `false` | Allows to configure DMS REST APIs usage | no | - |
| `spring.cloud.gcp.datastore.namespace` | ex `namespace` | Allows to configure Datastore namespace for DMS service properties storage | no | - |
| `DMS_API_BASE` | ex `http://localhost:8081/api/file/v2/files` | Only for local usage. Allows to override DMS service base url value from Datastore. | no | - |
## Configuring mappers' Datasources
When using non-Google-Cloud-native technologies, property sets must be defined on the Partition service as part of PartitionInfo for each Tenant.
They are specific to each storage technology:
#### for OSM - Postgres:
**prefix:** `osm.postgres`
It can be overridden by:
- through the Spring Boot property `osm.postgres.partitionPropertiesPrefix`
- environment variable `OSM_POSTGRES_PARTITIONPROPERTIESPREFIX`
**Propertyset:**
| Property | Description |
| --- | --- |
| osm.postgres.datasource.url | server URL |
| osm.postgres.datasource.username | username |
| osm.postgres.datasource.password | password |
<details><summary>Example of a definition for a single tenant</summary>
```
curl -L -X PATCH 'https://dev.osdu.club/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
"properties": {
"osm.postgres.datasource.url": {
"sensitive": false,
"value": "jdbc:postgresql://35.239.205.90:5432/postgres"
},
"osm.postgres.datasource.username": {
"sensitive": false,
"value": "osm_poc"
},
"osm.postgres.datasource.password": {
"sensitive": true,
"value": "osm_poc"
}
}
}'
```
</details>
#### for OBM - Minio:
**prefix:** `obm.minio`
It can be overridden by:
- through the Spring Boot property `obm.minio.partitionPropertiesPrefix`
- environment variable `OBM_MINIO_PARTITIONPROPERTIESPREFIX`
**Propertyset** (for two types of connection: messaging and admin operations):
| Property | Description |
| --- | --- |
| obm.minio.endpoint | - url |
| obm.minio.credentials.access.key | - username |
| obm.minio.credentials.secret.key | - password |
<details><summary>Example of a single tenant definition</summary>
```
curl -L -X PATCH 'https://dev.osdu.club/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
"properties": {
"obm.minio.endpoint": {
"sensitive": false,
"value": "localhost"
},
"obm.minio.credentials.access.key": {
"sensitive": false,
"value": "minioadmin"
},
"obm.minio.credentials.secret.key": {
"sensitive": false,
"value": "secret"
}
}
}'
```
</details>
## For postgres only. Schema configuration
```
CREATE TABLE public."DmsServiceProperties"(
id text COLLATE pg_catalog."default" NOT NULL,
pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
data jsonb NOT NULL,
CONSTRAINT DmsServiceProperties_id UNIQUE (id)
);
CREATE INDEX DmsServiceProperties_datagin ON public."DmsServiceProperties" USING GIN (data);
```
Example of filling table with DMS providers:
```
INSERT INTO public."DmsServiceProperties"(
id, data)
VALUES ('dataset--File.*', '{
"apiKey": "",
"datasetKind": "dataset--File.*",
"isStorageAllowed": true,
"dmsServiceBaseUrl": "https://localhost/api/file/v2/files",
"isStagingLocationSupported": true
}');
INSERT INTO public."DmsServiceProperties"(
id, data)
VALUES (
'dataset--FileCollection.*', '{
"apiKey": "",
"datasetKind": "dataset--FileCollection.*",
"isStorageAllowed": true,
"dmsServiceBaseUrl": "",
"isStagingLocationSupported": false
}');
```
### Run Locally
Check that maven is installed:
......
......@@ -9,4 +9,4 @@ ENV PORT $PORT
# Copy the jar to the production image from the builder stage.
COPY provider/dataset-${PROVIDER_NAME}/target/dataset-${PROVIDER_NAME}-*.jar dataset-${PROVIDER_NAME}.jar
# Run the web service on container startup.
CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=${PORT} -jar /app/dataset-${PROVIDER_NAME}.jar
CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=${PORT} -Dlog4j.formatMsgNoLookups=true -jar /app/dataset-${PROVIDER_NAME}.jar
......@@ -56,6 +56,16 @@
<artifactId>spring-cloud-gcp-starter-data-datastore</artifactId>
<version>1.2.5.RELEASE</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>osm</artifactId>
<version>0.13.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>obm</artifactId>
<version>0.13.1-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
......
......@@ -32,11 +32,9 @@ import org.springframework.context.annotation.FilterType;
@ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = {
@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = {DmsClientFactory.class,
DatasetApplication.class, IStorageFactory.class})})
@EnableDatastoreRepositories
public class DatasetApplicationGCP {
public static void main(String[] args) {
SpringApplication.run(DatasetApplicationGCP.class, args);
}
}
......@@ -19,13 +19,13 @@ package org.opengroup.osdu.dataset.provider.gcp.cache;