Merge branch 'master' of...

Merge branch 'master' of https://community.opengroup.org/osdu/platform/system/dataset into file_collection

.gitlab-ci.yml

@@ -9,7 +9,7 @@ variables:
   IBM_BUILD_SUBDIR: provider/dataset-ibm
   IBM_INT_TEST_SUBDIR: testing/dataset-test-ibm
 
-   # --- osdu-gcp specific variables ---
+  # --- osdu-gcp specific variables ---
   OSDU_GCP_SERVICE: dataset
   OSDU_GCP_VENDOR: gcp
   OSDU_GCP_APPLICATION_NAME: dataset
@@ -18,7 +18,7 @@ variables:
   OSDU_GCP_TEST_SUBDIR: testing/$OSDU_GCP_SERVICE-test-$OSDU_GCP_VENDOR
   OSDU_GCP_HELM_PACKAGE_CHARTS: "devops/gcp/deploy devops/gcp/configmap"
   OSDU_GCP_HELM_CONFIG_SERVICE_VARS: "--set data.google_audiences=$GOOGLE_AUDIENCE --set data.redis_group_host=$REDIS_GROUP_HOST --set data.file_dms_bucket=$OSDU_GCP_FILE_DMS_BUCKET --set data.expiration_days=$OSDU_GCP_EXPIRATION_DAYS"
-  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=workload-identity-dataset"
+  OSDU_GCP_HELM_DEPLOYMENT_SERVICE_VARS: "--set data.image=$CI_REGISTRY_IMAGE/osdu-gcp:$CI_COMMIT_SHORT_SHA --set data.serviceAccountName=$OSDU_GCP_SERVICE-k8s"
   OSDU_GCP_HELM_CONFIG_SERVICE: dataset-config
   OSDU_GCP_HELM_DEPLOYMENT_SERVICE: dataset-deploy
 
@@ -37,14 +37,14 @@ include:
   - project: "osdu/platform/ci-cd-pipelines"
     file: "scanners/fossa-maven.yml"
 
-  - project: 'osdu/platform/ci-cd-pipelines'
-    file: 'cloud-providers/aws-global.yml'
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/aws-global.yml"
 
-  - project: 'osdu/platform/ci-cd-pipelines'
-    file: 'cloud-providers/aws-maven.yml'
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/aws-maven.yml"
 
-  - project: 'osdu/platform/ci-cd-pipelines'
-    file: 'cloud-providers/azure.yml'
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/azure.yml"
 
   - project: "osdu/platform/ci-cd-pipelines"
     file: "cloud-providers/ibm.yml"
@@ -52,9 +52,8 @@ include:
   - project: "osdu/platform/ci-cd-pipelines"
     file: "scanners/gitlab-ultimate.yml"
 
-  - project: 'osdu/platform/ci-cd-pipelines'
-    file: 'cloud-providers/osdu-gcp-gke.yml'
-
+  - project: "osdu/platform/ci-cd-pipelines"
+    file: "cloud-providers/osdu-gcp-gke.yml"
 
 # disable the eslint scanner
 # I think this is being generated from the presence of an HTML file, but there

dataset-core/pom.xml

@@ -34,11 +34,8 @@
 		<java.version>1.8</java.version>
 		<maven.compiler.target>1.8</maven.compiler.target>
 		<maven.compiler.source>1.8</maven.compiler.source>
-		<springfox-version>2.7.0</springfox-version>
 		<tomcat_embed_core_version>9.0.54</tomcat_embed_core_version>
-		<!--
-		<springfox-version>2.9.2</springfox-version>
-		-->
+		<log4j2.version>2.17.0</log4j2.version>
 	</properties>
 
 	<dependencies>
@@ -79,8 +76,35 @@
 					<groupId>org.springframework.boot</groupId>
 					<artifactId>spring-boot-starter-tomcat</artifactId>
 				</exclusion>
-  			</exclusions>
+				<exclusion>
+					<groupId>org.apache.logging.log4j</groupId>
+					<artifactId>log4j-to-slf4j</artifactId>
+				</exclusion>
+				<exclusion>
+					<groupId>org.apache.logging.log4j</groupId>
+					<artifactId>log4j-api</artifactId>
+				</exclusion>
+			</exclusions>
+		</dependency>
+		
+		<dependency>
+			<groupId>org.apache.logging.log4j</groupId>
+			<artifactId>log4j-to-slf4j</artifactId>
+			<version>${log4j2.version}</version>
+			<exclusions>
+				<exclusion>
+					<groupId>org.apache.logging.log4j</groupId>
+					<artifactId>log4j-api</artifactId>
+				</exclusion>
+			</exclusions>
 		</dependency>
+		
+		<dependency>
+			<groupId>org.apache.logging.log4j</groupId>
+			<artifactId>log4j-api</artifactId>
+			<version>${log4j2.version}</version>
+		</dependency>
+		
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-security</artifactId>
@@ -148,18 +172,6 @@
 			<version>3.0.0</version>
 			<scope>test</scope>
 		</dependency>
-
-		<!-- swagger dependencies -->
-		<dependency>
-			<groupId>io.springfox</groupId>
-			<artifactId>springfox-swagger2</artifactId>
-			<version>${springfox-version}</version>
-		</dependency>
-		<dependency>
-			<groupId>io.springfox</groupId>
-			<artifactId>springfox-swagger-ui</artifactId>
-			<version>${springfox-version}</version>
-		</dependency>
     </dependencies>
 
 	<build>

dataset-core/src/main/java/org/opengroup/osdu/dataset/swagger/SpringfoxSwaggerHostResolver.java

+package org.opengroup.osdu.dataset.swagger;
+
+import io.swagger.v3.oas.models.OpenAPI;
+import io.swagger.v3.oas.models.servers.Server;
+import org.springframework.core.Ordered;
+import org.springframework.core.annotation.Order;
+import org.springframework.stereotype.Component;
+import springfox.documentation.oas.web.OpenApiTransformationContext;
+import springfox.documentation.oas.web.WebMvcOpenApiTransformationFilter;
+import springfox.documentation.spi.DocumentationType;
+
+import javax.servlet.http.HttpServletRequest;
+
+@Component
+@Order(Ordered.LOWEST_PRECEDENCE)
+public class SpringfoxSwaggerHostResolver implements WebMvcOpenApiTransformationFilter {
+
+    @Override
+    public boolean supports(DocumentationType delimiter) {
+        return delimiter == DocumentationType.OAS_30;
+    }
+
+    @Override
+    public OpenAPI transform(OpenApiTransformationContext<HttpServletRequest> context) {
+        OpenAPI swagger = context.getSpecification();
+
+        Server server = swagger.getServers().get(0);
+        if (server.getUrl().contains(":443")) {
+            // via the gateway
+            server.setUrl(server.getUrl().replace(":443",""));
+        }
+
+        return swagger;
+    }
+}
\ No newline at end of file

dataset-core/src/main/java/org/opengroup/osdu/dataset/swagger/SwaggerDocumentationConfig.java

@@ -19,25 +19,28 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Profile;
-import springfox.documentation.builders.ParameterBuilder;
+
 import springfox.documentation.builders.PathSelectors;
 import springfox.documentation.builders.RequestHandlerSelectors;
-import springfox.documentation.schema.ModelRef;
+import springfox.documentation.builders.RequestParameterBuilder;
+import springfox.documentation.oas.annotations.EnableOpenApi;
 import springfox.documentation.service.ApiKey;
 import springfox.documentation.service.AuthorizationScope;
+import springfox.documentation.service.ParameterType;
+import springfox.documentation.service.RequestParameter;
 import springfox.documentation.service.SecurityReference;
 import springfox.documentation.spi.DocumentationType;
 import springfox.documentation.spi.service.contexts.SecurityContext;
 import springfox.documentation.spring.web.plugins.Docket;
-import springfox.documentation.swagger2.annotations.EnableSwagger2;
-import springfox.documentation.service.Parameter;
+
+
 
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 
 @Configuration
-@EnableSwagger2
+@EnableOpenApi
 @Profile("!noswagger")
 public class SwaggerDocumentationConfig {
     public static final String AUTHORIZATION_HEADER = "Authorization";
@@ -45,26 +48,23 @@ public class SwaggerDocumentationConfig {
 
     @Bean
     public Docket api() {
-        ParameterBuilder builder = new ParameterBuilder();
-        List<Parameter> parameters = new ArrayList<>();
+    	RequestParameterBuilder builder = new RequestParameterBuilder();
+    	List<RequestParameter> parameters = new ArrayList<>();
+
         builder.name(DpsHeaders.DATA_PARTITION_ID)
                 .description("tenant")
-                .defaultValue("common")
-                .modelRef(new ModelRef("string"))
-                .parameterType("header")
+                .in(ParameterType.HEADER)
                 .required(true)
                 .build();
         parameters.add(builder.build());
         builder.name("frame-of-reference")
                 .description("reference")
-                .defaultValue("none")
-                .modelRef(new ModelRef("string"))
-                .parameterType("header")
+                .in(ParameterType.HEADER)
                 .required(true)
                 .build();
         parameters.add(builder.build());
-        return new Docket(DocumentationType.SWAGGER_2)
-                .globalOperationParameters(parameters)
+        return new Docket(DocumentationType.OAS_30)
+            	.globalRequestParameters(parameters)
                 .select()
                 .apis(RequestHandlerSelectors.basePackage("org.opengroup.osdu.dataset.api"))
                 .build()
@@ -73,13 +73,13 @@ public class SwaggerDocumentationConfig {
     }
 
     private ApiKey apiKey() {
-        return new ApiKey("JWT", AUTHORIZATION_HEADER, "header");
+    	 return new ApiKey(AUTHORIZATION_HEADER, AUTHORIZATION_HEADER, "header");
     }
 
     private SecurityContext securityContext() {
         return SecurityContext.builder()
                 .securityReferences(defaultAuth())
-                .forPaths(PathSelectors.regex(DEFAULT_INCLUDE_PATTERN))
+                .operationSelector(o -> PathSelectors.regex(DEFAULT_INCLUDE_PATTERN).test(o.requestMappingPattern()))
                 .build();
     }
 
@@ -89,6 +89,6 @@ public class SwaggerDocumentationConfig {
         AuthorizationScope[] authorizationScopes =
                 new AuthorizationScope[]{authorizationScope};
         return Collections.singletonList(
-                new SecurityReference("JWT", authorizationScopes));
+        		 new SecurityReference(AUTHORIZATION_HEADER, authorizationScopes));
     }
 }

pom.xml

@@ -26,12 +26,14 @@
 		<maven.compiler.source>1.8</maven.compiler.source>
 		<docker.image.prefix>opendes</docker.image.prefix>
 		<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
-		<os-core-common.version>0.12.0</os-core-common.version>
+		<os-core-common.version>0.13.0-rc4</os-core-common.version>
 		<jackson.version>2.11.2</jackson.version>
 		<netty.version>4.1.51.Final</netty.version>
 		<snakeyaml.version>1.26</snakeyaml.version>
 		<commons-codec.version>1.14</commons-codec.version>
 		<nimbusds.version>7.9</nimbusds.version>
+		<log4j2.version>2.17.0</log4j2.version>
+		<springfox.version>3.0.0</springfox.version>
 	</properties>
 
 	<licenses>
@@ -53,6 +55,21 @@
 				<type>pom</type>
 				<scope>import</scope>
 			</dependency> -->
+			<dependency>
+				<groupId>org.apache.logging.log4j</groupId>
+				<artifactId>log4j-to-slf4j</artifactId>
+				<version>${log4j2.version}</version>
+			</dependency>
+			<dependency>
+				<groupId>org.apache.logging.log4j</groupId>
+				<artifactId>log4j-api</artifactId>
+				<version>${log4j2.version}</version>
+			</dependency>
+			<dependency>
+				<groupId>org.opengroup.osdu</groupId>
+				<artifactId>os-core-common</artifactId>
+				<version>${os-core-common.version}</version>
+			</dependency>
 			<dependency>
 				<groupId>org.springframework.boot</groupId>
 				<artifactId>spring-boot-dependencies</artifactId>
@@ -85,11 +102,6 @@
 				<artifactId>snakeyaml</artifactId>
 				<version>${snakeyaml.version}</version>
 			</dependency> -->
-			<dependency>
-				<groupId>org.opengroup.osdu</groupId>
-				<artifactId>os-core-common</artifactId>
-				<version>${os-core-common.version}</version>
-			</dependency>
 		</dependencies>
 	</dependencyManagement>
 
@@ -100,6 +112,13 @@
 			<version>1.18.8</version>
 			<scope>provided</scope>
 		</dependency>
+		<!-- swagger dependency -->
+		<dependency>
+      		<groupId>io.springfox</groupId>
+      		<artifactId>springfox-boot-starter</artifactId>
+      		<version>${springfox.version}</version>
+    	</dependency>
+		
 	</dependencies>
 
 	<profiles>

provider/dataset-aws/pom.xml

@@ -58,7 +58,7 @@
         <dependency>
             <groupId>org.opengroup.osdu.core.aws</groupId>
             <artifactId>os-core-lib-aws</artifactId>
-            <version>0.12.0</version>
+            <version>0.13.0-rc3</version>
         </dependency>
         <dependency>
             <groupId>org.opengroup.osdu</groupId>

provider/dataset-azure/pom.xml

@@ -37,8 +37,8 @@
         <java.version>1.8</java.version>
         <maven.compiler.target>1.8</maven.compiler.target>
         <maven.compiler.source>1.8</maven.compiler.source>
-        <osdu.corelibazure.version>0.12.0</osdu.corelibazure.version>
-        <osdu.oscorecommon-version>0.12.0</osdu.oscorecommon-version>
+        <osdu.corelibazure.version>0.13.0-rc6</osdu.corelibazure.version>
+        <osdu.oscorecommon-version>0.13.0-rc4</osdu.oscorecommon-version>
         <nimbus-jose-jwt.version>7.9</nimbus-jose-jwt.version>
     </properties>
 

provider/dataset-azure/src/main/java/org/opengroup/osdu/dataset/provider/azure/security/AADSecurityConfig.java

@@ -42,11 +42,13 @@ public class AADSecurityConfig extends WebSecurityConfigurerAdapter {
                 .authorizeRequests()
                 .antMatchers("/", "/index.html",
                         "/v2/api-docs",
+                        "/v3/api-docs",
                         "/configuration/ui",
                         "/swagger-resources/**",
                         "/configuration/security",
                         "/swagger",
                         "/swagger-ui.html",
+                        "/swagger-ui/**",
                         "/info",
                         "/webjars/**").permitAll()
                 .anyRequest().authenticated()

provider/dataset-gcp/README.md

@@ -4,30 +4,210 @@
 These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.
 
 ### Prerequisites
-Pre-requisites
 
 * GCloud SDK with java (latest version)
 * JDK 8
 * Lombok 1.16 or later
 * Maven
 
-### Installation
-In order to run the service locally or remotely, you will need to have the following environment variables defined.
+# Features of implementation
+This is a universal solution created using EPAM OSM and OBM mappers technology.
+It allows you to work with various implementations of KV stores and Blob stores.
+
+## Limitations of the current version
+
+In the current version, the mappers are equipped with several drivers to the stores:
+
+- OSM (mapper for KV-data): Google Datastore; Postgres
+- OBM (mapper to Blob stores): Google Cloud Storage (GCS); MinIO
+
+## Extensibility
+
+To use any other store or message broker, implement a driver for it. With an extensible set of drivers, the solution is unrestrictedly universal and portable without modification to the main code.
+
+Mappers support "multitenancy" with flexibility in how it is implemented.
+They switch between datasources of different tenants due to the work of a bunch of classes that implement the following interfaces:
+
+- Destination - takes a description of the current context, e.g., "data-partition-id = opendes"
+- DestinationResolver – accepts Destination, finds the resource, connects, and returns Resolution
+- DestinationResolution – contains a ready-made connection, the mapper uses it to get to data
+
+## Mapper tuning mechanisms
+
+This service uses specific implementations of DestinationResolvers based on the tenant information provided by the OSDU Partition service.
+- for Google Datastore: osm/config/DsTenantDestinationResolver.java
+- for Postgres: osm/config/PgTenantDestinationResolver.java
+
+#### Their algorithms are as follows:
+- incoming Destination carries data-partition-id
+- resolver accesses the Partition service and gets PartitionInfo
+- from PartitionInfo resolver retrieves properties for the connection: URL, username, password etc.
+- resolver creates a data source, connects to the resource, remembers the datasource
+- resolver gives the datasource to the mapper in the Resolution object
+- Google Cloud resolvers do not receive special properties from the Partition service for connection, 
+because the location of the resources is unambiguously known - they are in the GCP project. 
+And credentials are also not needed - access to data is made on behalf of the Google Identity SA
+under which the service itself is launched. Therefore, resolver takes only 
+the value of the **projectId** property from PartitionInfo and uses it to connect to a resource 
+in the corresponding GCP project.
+
+# Configuration
+
+## Service Configuration
+
+Define the following environment variables.
+Most of them are common to all hosting environments, but there are properties that are only necessary when running in Google Cloud.
+
+#### Common properties for all environments
 
 | name | value | description | sensitive? | source |
 | ---  | ---   | ---         | ---        | ---    |
-| `GCP_SCHEMA_API` | ex `https://os-schema-jvmvia5dea-uc.a.run.app/api/schema-service/v1` | Schema API endpoint | no | output of infrastructure deployment |
-| `GCP_STORAGE_API` | ex `https://os-storage-jvmvia5dea-uc.a.run.app/api/storage/v2` | Storage API endpoint | no | output of infrastructure deployment |
+| `SCHEMA_API` | ex `https://os-schema-jvmvia5dea-uc.a.run.app/api/schema-service/v1` | Schema API endpoint | no | output of infrastructure deployment |
+| `STORAGE_API` | ex `https://os-storage-jvmvia5dea-uc.a.run.app/api/storage/v2` | Storage API endpoint | no | output of infrastructure deployment |
 | `AUTHORIZE_API` | ex `https://os-entitlements-gcp-jvmvia5dea-uc.a.run.app/entitlements/v1` | Entitlements API endpoint | no | output of infrastructure deployment |
+| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
 | `FILE_DMS_BUCKET` | ex `file-dms-bucket` | File bucket name postfix (full name represent by project-id + partition-id + GCP_FILE_DMS_BUCKET ex `osdu-cicd-epam-opendes-file-dms-bucket`) | no | output of infrastructure deployment |
 | `EXPIRATION_DAYS` | ex `1` | expiration for signed urls & connection strings | no |  |
 | `REDIS_GROUP_HOST` |  ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
 | `REDIS_GROUP_PORT` |  ex `1111` | Redis port | no | https://console.cloud.google.com/memorystore/redis/instances |
+| `osdu.dataset.config.useRestDms` | `true` OR `false` | Allows to configure *DMS REST APIs usage* | no | - |
+| `DMS_API_BASE` | ex `http://localhost:8081/api/file/v2/files` | *Only for local usage.* Allows to override DMS service base url value from Datastore.  | no | - |
+
+#### For Mappers, to activate drivers
+| name | value | description | sensitive? | source |
+| ---  | ---   | ---         | ---        | ---    |
+| `OSMDRIVER` | `postgres` OR `datastore` | Osm driver mode that defines which KV storage will be used | no | - |
+| `OBMDRIVER` | `gcs` OR `minio` | Obm driver mode that defines which object storage will be used | no | - |
+
+#### For Google Cloud only
+| name | value | description | sensitive? | source |
+| ---  | ---   | ---         | ---        | ---    |
 | `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
-| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
-| `osdu.dataset.config.useRestDms` | `true` OR `false` | Allows to configure DMS REST APIs usage | no | - |
 | `spring.cloud.gcp.datastore.namespace` | ex `namespace` | Allows to configure Datastore namespace for DMS service properties storage | no | - |
-| `DMS_API_BASE` | ex `http://localhost:8081/api/file/v2/files` | Only for local usage. Allows to override DMS service base url value from Datastore.  | no | - |
+
+## Configuring mappers' Datasources
+When using non-Google-Cloud-native technologies, property sets must be defined on the Partition service as part of PartitionInfo for each Tenant.
+
+They are specific to each storage technology:
+#### for OSM - Postgres:
+**prefix:** `osm.postgres`
+It can be overridden by:
+- through the Spring Boot property `osm.postgres.partitionPropertiesPrefix`
+- environment variable `OSM_POSTGRES_PARTITIONPROPERTIESPREFIX`
+
+**Propertyset:**
+
+| Property | Description |
+| --- | --- |
+| osm.postgres.datasource.url | server URL |
+| osm.postgres.datasource.username | username |
+| osm.postgres.datasource.password | password |
+
+<details><summary>Example of a definition for a single tenant</summary>
+
+```
+
+curl -L -X PATCH 'https://dev.osdu.club/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
+  "properties": {
+    "osm.postgres.datasource.url": {
+      "sensitive": false,
+      "value": "jdbc:postgresql://35.239.205.90:5432/postgres"
+    },
+    "osm.postgres.datasource.username": {
+      "sensitive": false,
+      "value": "osm_poc"
+    },
+    "osm.postgres.datasource.password": {
+      "sensitive": true,
+      "value": "osm_poc"
+    }
+  }
+}'
+
+```
+
+</details>
+
+#### for OBM - Minio:
+**prefix:** `obm.minio`
+It can be overridden by:
+
+- through the Spring Boot property `obm.minio.partitionPropertiesPrefix`
+- environment variable `OBM_MINIO_PARTITIONPROPERTIESPREFIX`
+
+**Propertyset** (for two types of connection: messaging and admin operations):
+
+| Property | Description |
+| --- | --- |
+| obm.minio.endpoint | - url |
+| obm.minio.credentials.access.key | - username |
+| obm.minio.credentials.secret.key | - password |
+
+<details><summary>Example of a single tenant definition</summary>
+
+```
+
+curl -L -X PATCH 'https://dev.osdu.club/api/partition/v1/partitions/opendes' -H 'data-partition-id: opendes' -H 'Authorization: Bearer ...' -H 'Content-Type: application/json' --data-raw '{
+  "properties": {
+    "obm.minio.endpoint": {
+      "sensitive": false,
+      "value": "localhost"
+    },
+    "obm.minio.credentials.access.key": {
+      "sensitive": false,
+      "value": "minioadmin"
+    },
+    "obm.minio.credentials.secret.key": {
+      "sensitive": false,
+      "value": "secret"
+    }
+  }
+}'
+
+```
+
+</details>
+
+## For postgres only. Schema configuration
+
+```
+
+CREATE TABLE public."DmsServiceProperties"(
+id text COLLATE pg_catalog."default" NOT NULL,
+pk bigint NOT NULL GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
+data jsonb NOT NULL,
+CONSTRAINT DmsServiceProperties_id UNIQUE (id)
+);
+CREATE INDEX DmsServiceProperties_datagin ON public."DmsServiceProperties" USING GIN (data);
+
+```
+
+Example of filling table with DMS providers:
+
+```
+
+INSERT INTO public."DmsServiceProperties"(
+id, data)
+VALUES ('dataset--File.*', '{
+"apiKey": "",
+"datasetKind": "dataset--File.*",
+"isStorageAllowed": true,
+"dmsServiceBaseUrl": "https://localhost/api/file/v2/files",
+"isStagingLocationSupported": true
+}');
+
+INSERT INTO public."DmsServiceProperties"(
+id, data)
+VALUES (
+'dataset--FileCollection.*', '{
+"apiKey": "",
+"datasetKind": "dataset--FileCollection.*",
+"isStorageAllowed": true,
+"dmsServiceBaseUrl": "",
+"isStagingLocationSupported": false
+}');
+
+```
 
 ### Run Locally
 Check that maven is installed:

provider/dataset-gcp/cloudbuild/Dockerfile.cloudbuild

@@ -9,4 +9,4 @@ ENV PORT $PORT
 # Copy the jar to the production image from the builder stage.
 COPY provider/dataset-${PROVIDER_NAME}/target/dataset-${PROVIDER_NAME}-*.jar dataset-${PROVIDER_NAME}.jar
 # Run the web service on container startup.
-CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=${PORT} -jar /app/dataset-${PROVIDER_NAME}.jar
+CMD java -Djava.security.egd=file:/dev/./urandom -Dserver.port=${PORT} -Dlog4j.formatMsgNoLookups=true -jar /app/dataset-${PROVIDER_NAME}.jar

provider/dataset-gcp/pom.xml

@@ -56,6 +56,16 @@
 			<artifactId>spring-cloud-gcp-starter-data-datastore</artifactId>
 			<version>1.2.5.RELEASE</version>
 		</dependency>
+		<dependency>
+			<groupId>org.opengroup.osdu</groupId>
+			<artifactId>osm</artifactId>
+			<version>0.13.0-SNAPSHOT</version>
+		</dependency>
+		<dependency>
+			<groupId>org.opengroup.osdu</groupId>
+			<artifactId>obm</artifactId>
+			<version>0.13.1-SNAPSHOT</version>
+		</dependency>
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>

provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/DatasetApplicationGCP.java

@@ -32,11 +32,9 @@ import org.springframework.context.annotation.FilterType;
 @ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = {
 	@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = {DmsClientFactory.class,
 		DatasetApplication.class, IStorageFactory.class})})
-@EnableDatastoreRepositories
 public class DatasetApplicationGCP {
 
 	public static void main(String[] args) {
 		SpringApplication.run(DatasetApplicationGCP.class, args);
 	}
-
 }

provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/cache/GroupCache.java

@@ -19,13 +19,13 @@ package org.opengroup.osdu.dataset.provider.gcp.cache;