Merge branch 'gcp-remap-dataset-to-use-file-service' into 'master'

Remap Dataset To Use File Service (GONRG-2918)

See merge request !121

.gitlab-ci.yml

@@ -67,3 +67,5 @@ osdu-gcp-test:
     PROVIDER_KEY: GCP
     KIND_SUBTYPE: DatasetTest
     LEGAL_TAG: datatest-int-tag
+    LEGAL_HOST: $OSDU_GCP_LEGALTAG_API/
+    GCP_STORAGE_PERSISTENT_AREA: persisten-area

dataset-core/src/main/java/org/opengroup/osdu/dataset/dms/DmsRestService.java

@@ -19,6 +19,8 @@ package org.opengroup.osdu.dataset.dms;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.net.URISyntaxException;
+import java.util.List;
 import lombok.RequiredArgsConstructor;
 import org.apache.http.HttpStatus;
 import org.apache.http.client.utils.URIBuilder;
@@ -33,8 +35,6 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
 import org.opengroup.osdu.dataset.model.request.GetDatasetRegistryRequest;
 import org.opengroup.osdu.dataset.model.response.GetDatasetRetrievalInstructionsResponse;
 import org.opengroup.osdu.dataset.model.response.GetDatasetStorageInstructionsResponse;
-import java.net.URISyntaxException;
-import java.util.List;
 
 @RequiredArgsConstructor
 public class DmsRestService implements IDmsProvider {

provider/dataset-gcp/README.md

@@ -25,7 +25,9 @@ In order to run the service locally or remotely, you will need to have the follo
 | `REDIS_GROUP_PORT` |  ex `1111` | Redis port | no | https://console.cloud.google.com/memorystore/redis/instances |
 | `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
 | `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
-
+| `osdu.dataset.config.useRestDms` | `true` OR `false` | Allows to configure DMS REST APIs usage | no | - |
+| `spring.cloud.gcp.datastore.namespace` | ex `namespace` | Allows to configure Datastore namespace for DMS service properties storage | no | - |
+| `DMS_API_BASE` | ex `http://localhost:8081/api/file/v2/files` | Only for local usage. Allows to override DMS service base url value from Datastore.  | no | - |
 
 ### Run Locally
 Check that maven is installed:
@@ -119,13 +121,14 @@ cd provider/dataset-gcp && mvn spring-boot:run
  | `KIND_SUBTYPE` | `DatasetTest` | Kind subtype that will be used in int tests, schema creation automated , result kind will be `TENANT_NAME::wks-test:dataset--FileCollection.KIND_SUBTYPE:1.0.0`| no | - |
  | `LEGAL_TAG` | `public-usa-dataset-1` | Legal tag name, if tag with that name doesn't exist then it will be created during preparing step | no | - |
  | `GCLOUD_PROJECT` | `osdu-cicd-epam` | Project id | no | - |
-
+ | `GCP_STORAGE_PERSISTENT_AREA` | ex `persistent-area` | persistent area bucket(will be concatenated with project id ex `osdu-cicd-epam-persistent-area` | no | output of infrastructure deployment |
+ | `LEGAL_HOST` | ex `https://os-legal-jvmvia5dea-uc.a.run.app/api/legal/v1/` | Legal API endpoint | no | output of infrastructure deployment |
 
  **Entitlements configuration for integration accounts**
  
  | INTEGRATION_TESTER | 
  | ---  | 
- | users<br/>service.entitlements.user<br/>service.storage.admin<br/>service.legal.user<br/>service.search.user<br/>service.delivery.viewer | 
+ | users<br/>service.entitlements.user<br/>service.storage.admin<br/>service.legal.user<br/>service.search.user<br/>service.delivery.viewer<br/>service.dataset.viewers<br/>service.dataset.editors | 
  
  **Cloud roles configuration for integration accounts**
  

provider/dataset-gcp/pom.xml

@@ -51,6 +51,11 @@
 			<artifactId>core-lib-gcp</artifactId>
 			<version>0.9.0</version>
 		</dependency>
+		<dependency>
+			<groupId>org.springframework.cloud</groupId>
+			<artifactId>spring-cloud-gcp-starter-data-datastore</artifactId>
+			<version>1.2.5.RELEASE</version>
+		</dependency>
 		<dependency>
 			<groupId>junit</groupId>
 			<artifactId>junit</artifactId>

provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/DatasetApplicationGCP.java

@@ -23,6 +23,7 @@ import org.opengroup.osdu.dataset.di.DmsClientFactory;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
+import org.springframework.cloud.gcp.data.datastore.repository.config.EnableDatastoreRepositories;
 import org.springframework.context.annotation.ComponentScan;
 import org.springframework.context.annotation.FilterType;
 
@@ -31,6 +32,7 @@ import org.springframework.context.annotation.FilterType;
 @ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = {
 	@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = {DmsClientFactory.class,
 		DatasetApplication.class, IStorageFactory.class})})
+@EnableDatastoreRepositories
 public class DatasetApplicationGCP {
 
 	public static void main(String[] args) {

provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/config/GcpPropertiesConfig.java

@@ -36,4 +36,6 @@ public class GcpPropertiesConfig {
 	private String redisGroupHost;
 
 	private Integer redisGroupPort;
+
+	private String dmsApiBase;
 }

provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/dms/DatasetDmsServiceMapImpl.java

@@ -20,9 +20,13 @@ package org.opengroup.osdu.dataset.provider.gcp.dms;
 import java.util.HashMap;
 import java.util.Map;
 import lombok.RequiredArgsConstructor;
+import org.apache.commons.lang3.StringUtils;
 import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
+import org.opengroup.osdu.dataset.provider.gcp.config.GcpPropertiesConfig;
 import org.opengroup.osdu.dataset.provider.gcp.model.dataset.DataSetType;
+import org.opengroup.osdu.dataset.provider.gcp.model.dataset.DmsServicePropertiesEntity;
 import org.opengroup.osdu.dataset.provider.gcp.model.dataset.GcpDmsServiceProperties;
+import org.opengroup.osdu.dataset.provider.gcp.repository.DmsServicePropertiesEntityRepository;
 import org.opengroup.osdu.dataset.provider.interfaces.IDatasetDmsServiceMap;
 import org.springframework.stereotype.Service;
 
@@ -30,20 +34,34 @@ import org.springframework.stereotype.Service;
 @RequiredArgsConstructor
 public class DatasetDmsServiceMapImpl implements IDatasetDmsServiceMap {
 
-	private final Map<String, DmsServiceProperties> resourceTypeToDmsServiceMap = new HashMap<>();
+  private final DmsServicePropertiesEntityRepository dmsServicePropertiesEntityRepository;
+  private final GcpPropertiesConfig gcpPropertiesConfig;
 
-	@Override
-	public Map<String, DmsServiceProperties> getResourceTypeToDmsServiceMap() {
-		resourceTypeToDmsServiceMap.put(
-			"dataset--File.*",
-			new GcpDmsServiceProperties(DataSetType.FILE)
-		);
+  @Override
+  public Map<String, DmsServiceProperties> getResourceTypeToDmsServiceMap() {
+    Iterable<DmsServicePropertiesEntity> properties = this.dmsServicePropertiesEntityRepository.findAll();
 
-		resourceTypeToDmsServiceMap.put(
-			"dataset--FileCollection.*",
-			new GcpDmsServiceProperties(DataSetType.FILE_COLLECTION)
-		);
+    Map<String, DmsServiceProperties> resourceTypeToDmsServiceMap = new HashMap<>();
+
+    for (DmsServicePropertiesEntity entity : properties) {
+      String dmsApiBase = this.gcpPropertiesConfig.getDmsApiBase();
+      if (StringUtils.isEmpty(dmsApiBase)) {
+        dmsApiBase = entity.getDmsServiceBaseUrl();
+      }
+
+      DmsServiceProperties dmsServiceProperties = new DmsServiceProperties(dmsApiBase, entity.isStorageAllowed(),
+          entity.getApiKey(), entity.isStagingLocationSupported());
+      
+      if (StringUtils.containsIgnoreCase(entity.getDatasetKind(), "collection")) {
+        resourceTypeToDmsServiceMap.put(entity.getDatasetKind(), new GcpDmsServiceProperties(
+            DataSetType.FILE_COLLECTION, dmsServiceProperties));
+      } else {
+        resourceTypeToDmsServiceMap.put(entity.getDatasetKind(), new GcpDmsServiceProperties(
+            DataSetType.FILE, dmsServiceProperties));
+      }
+    }
+
+    return resourceTypeToDmsServiceMap;
+  }
 
-		return resourceTypeToDmsServiceMap;
-	}
 }

provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/dms/GcpDmsFactory.java

@@ -18,7 +18,10 @@
 package org.opengroup.osdu.dataset.provider.gcp.dms;
 
 import lombok.RequiredArgsConstructor;
+import org.opengroup.osdu.core.common.http.HttpClient;
+import org.opengroup.osdu.core.common.http.IHttpClient;
 import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.dataset.dms.DmsRestService;
 import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
 import org.opengroup.osdu.dataset.dms.IDmsFactory;
 import org.opengroup.osdu.dataset.dms.IDmsProvider;
@@ -28,10 +31,13 @@ import org.opengroup.osdu.dataset.provider.gcp.service.IFileService;
 @RequiredArgsConstructor
 public class GcpDmsFactory implements IDmsFactory {
 
-	private final IFileService fileDmsService;
+  private final IFileService fileDmsService;
 
-	@Override
-	public IDmsProvider create(DpsHeaders headers, DmsServiceProperties dmsServiceRoute) {
-		return new GcpDmsService(fileDmsService, (GcpDmsServiceProperties) dmsServiceRoute);
-	}
+  private IHttpClient httpClient = new HttpClient();
+
+  @Override
+  public IDmsProvider create(DpsHeaders headers, DmsServiceProperties dmsServiceRoute) {
+    return new GcpDmsService(fileDmsService, (GcpDmsServiceProperties) dmsServiceRoute,
+        new GcpDmsRestService(dmsServiceRoute, httpClient, headers));
+  }
 }

provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/dms/GcpDmsRestService.java

+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.dataset.provider.gcp.dms;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import java.net.URISyntaxException;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.utils.URIBuilder;
+import org.opengroup.osdu.core.common.http.HttpRequest;
+import org.opengroup.osdu.core.common.http.HttpResponse;
+import org.opengroup.osdu.core.common.http.IHttpClient;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.opengroup.osdu.dataset.dms.DmsRestService;
+import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
+import org.opengroup.osdu.dataset.model.response.GetDatasetStorageInstructionsResponse;
+
+public class GcpDmsRestService extends DmsRestService {
+
+  private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+  private final DmsServiceProperties dmsServiceProperties;
+  private final IHttpClient httpClient;
+  private final DpsHeaders headers;
+
+  public GcpDmsRestService(DmsServiceProperties dmsServiceProperties,
+      IHttpClient httpClient,
+      DpsHeaders headers) {
+    super(dmsServiceProperties, httpClient, headers);
+    this.dmsServiceProperties = dmsServiceProperties;
+    this.httpClient = httpClient;
+    this.headers = headers;
+  }
+
+  @Override
+  public GetDatasetStorageInstructionsResponse getStorageInstructions() {
+    String url = this.createUrl("/storageInstructions");
+    HttpResponse result = this.httpClient
+        .send(HttpRequest.post("").url(url).headers(this.headers.getHeaders()).build());
+    try {
+      return OBJECT_MAPPER.readValue(result.getBody(), GetDatasetStorageInstructionsResponse.class);
+    } catch (JsonProcessingException e) {
+      throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal Server Error",
+          e.getMessage(), e);
+    }
+  }
+
+  private String createUrl(String path) {
+    try {
+      URIBuilder uriBuilder = new URIBuilder(dmsServiceProperties.getDmsServiceBaseUrl());
+      uriBuilder.setPath(uriBuilder.getPath() + path);
+      return uriBuilder.build().normalize().toString();
+    } catch (URISyntaxException e) {
+      throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Invalid URL", e.getMessage(), e);
+    }
+  }
+
+}

provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/dms/GcpDmsService.java

@@ -17,12 +17,19 @@
 
 package org.opengroup.osdu.dataset.provider.gcp.dms;
 
+import java.util.ArrayList;
+import java.util.List;
 import lombok.RequiredArgsConstructor;
 import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.dms.model.CopyDmsRequest;
+import org.opengroup.osdu.core.common.dms.model.CopyDmsResponse;
+import org.opengroup.osdu.core.common.dms.model.DatasetRetrievalProperties;
+import org.opengroup.osdu.core.common.dms.model.RetrievalInstructionsResponse;
 import org.opengroup.osdu.core.common.model.http.AppException;
 import org.opengroup.osdu.dataset.dms.DmsException;
 import org.opengroup.osdu.dataset.dms.IDmsProvider;
 import org.opengroup.osdu.dataset.model.request.GetDatasetRegistryRequest;
+import org.opengroup.osdu.dataset.model.response.DatasetRetrievalDeliveryItem;
 import org.opengroup.osdu.dataset.model.response.GetDatasetRetrievalInstructionsResponse;
 import org.opengroup.osdu.dataset.model.response.GetDatasetStorageInstructionsResponse;
 import org.opengroup.osdu.dataset.provider.gcp.model.dataset.GcpDmsServiceProperties;
@@ -31,32 +38,61 @@ import org.opengroup.osdu.dataset.provider.gcp.service.IFileService;
 @RequiredArgsConstructor
 public class GcpDmsService implements IDmsProvider {
 
-	private final IFileService fileDmsService;
-
-	private final GcpDmsServiceProperties dmsServiceProperties;
-
-	@Override
-	public GetDatasetStorageInstructionsResponse getStorageInstructions() throws DmsException {
-		switch (dmsServiceProperties.getDataSetType()) {
-			case FILE:
-				return fileDmsService.getFileUploadInstructions();
-			case FILE_COLLECTION:
-				return fileDmsService.getCollectionUploadInstructions();
-			default:
-				throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "Invalid dataset provided");
-		}
-	}
-
-	@Override
-	public GetDatasetRetrievalInstructionsResponse getDatasetRetrievalInstructions(GetDatasetRegistryRequest request)
-		throws DmsException {
-		switch (dmsServiceProperties.getDataSetType()) {
-			case FILE:
-				return fileDmsService.getFileRetrievalInstructions(request);
-			case FILE_COLLECTION:
-				return fileDmsService.getCollectionRetrievalInstructions(request);
-			default:
-				throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "Invalid dataset provided");
-		}
-	}
+  private final IFileService fileDmsService;
+
+  private final GcpDmsServiceProperties dmsServiceProperties;
+
+  private final IDmsProvider dmsRestService;
+
+  @Override
+  public GetDatasetStorageInstructionsResponse getStorageInstructions() throws DmsException {
+    switch (dmsServiceProperties.getDataSetType()) {
+      case FILE:
+        return dmsRestService.getStorageInstructions();
+      case FILE_COLLECTION:
+        return fileDmsService.getCollectionUploadInstructions();
+      default:
+        throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request",
+            "Invalid dataset provided");
+    }
+  }
+
+  @Override
+  public GetDatasetRetrievalInstructionsResponse getDatasetRetrievalInstructions(
+      GetDatasetRegistryRequest request)
+      throws DmsException {
+    switch (dmsServiceProperties.getDataSetType()) {
+      case FILE:
+        RetrievalInstructionsResponse retrievalInstructions = dmsRestService.getRetrievalInstructions(request);
+        String providerKey = retrievalInstructions.getProviderKey();
+
+        List<DatasetRetrievalDeliveryItem> datasetRetrievalDeliveryItemList = new ArrayList<>();
+
+        for (DatasetRetrievalProperties properties : retrievalInstructions.getDatasets()) {
+          DatasetRetrievalDeliveryItem retrievalDeliveryItem = new DatasetRetrievalDeliveryItem(
+              properties.getDatasetRegistryId(), properties.getRetrievalProperties(),
+              providerKey);
+          datasetRetrievalDeliveryItemList.add(retrievalDeliveryItem);
+        }
+
+        return new GetDatasetRetrievalInstructionsResponse(datasetRetrievalDeliveryItemList);
+      case FILE_COLLECTION:
+        return fileDmsService.getCollectionRetrievalInstructions(request);
+      default:
+        throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request",
+            "Invalid dataset provided");
+    }
+  }
+
+  @Override
+  public RetrievalInstructionsResponse getRetrievalInstructions(GetDatasetRegistryRequest request)
+      throws DmsException {
+    return dmsRestService.getRetrievalInstructions(request);
+  }
+
+  @Override
+  public List<CopyDmsResponse> copyDmsToPersistentStorage(CopyDmsRequest copyDmsRequest)
+      throws DmsException {
+    return dmsRestService.copyDmsToPersistentStorage(copyDmsRequest);
+  }
 }

provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/model/dataset/DmsServicePropertiesEntity.java

+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.dataset.provider.gcp.model.dataset;
+
+import lombok.AllArgsConstructor;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+import org.springframework.cloud.gcp.data.datastore.core.mapping.Entity;
+import org.springframework.data.annotation.Id;
+
+@Entity(name = "DmsServiceProperties")
+@Data
+@NoArgsConstructor
+@AllArgsConstructor
+public class DmsServicePropertiesEntity {
+
+  @Id
+  private String datasetKind;
+
+  private String dmsServiceBaseUrl;
+
+  private boolean isStorageAllowed;
+
+  private String apiKey;
+
+  private boolean isStagingLocationSupported;
+}
+

provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/model/dataset/GcpDmsServiceProperties.java

@@ -27,5 +27,11 @@ import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
 @AllArgsConstructor
 public class GcpDmsServiceProperties extends DmsServiceProperties {
 
-	private DataSetType dataSetType;
+  private DataSetType dataSetType;
+
+  public GcpDmsServiceProperties(DataSetType dataSetType, DmsServiceProperties dmsServiceProperties) {
+    super(dmsServiceProperties.getDmsServiceBaseUrl(), dmsServiceProperties.isAllowStorage(),
+        dmsServiceProperties.getApiKey(), dmsServiceProperties.isStagingLocationSupported());
+    this.dataSetType = dataSetType;
+  }
 }

provider/dataset-gcp/src/main/java/org/opengroup/osdu/dataset/provider/gcp/repository/DmsServicePropertiesEntityRepository.java

+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.dataset.provider.gcp.repository;
+
+import org.opengroup.osdu.dataset.provider.gcp.model.dataset.DmsServicePropertiesEntity;
+import org.springframework.cloud.gcp.data.datastore.repository.DatastoreRepository;
+import org.springframework.stereotype.Repository;
+
+@Repository
+public interface DmsServicePropertiesEntityRepository extends
+    DatastoreRepository<DmsServicePropertiesEntity, String> {
+
+}

provider/dataset-gcp/src/main/resources/application.properties

@@ -36,3 +36,9 @@ LOG_PREFIX=${log.prefix}
 
 google.audiences=123.apps.googleusercontent.com
 partition.api=http://localhost:8081/api/partition/v1
+
+DMS_API_BASE=
+
+osdu.dataset.config.useRestDms=false
+
+spring.cloud.gcp.datastore.namespace=

provider/dataset-gcp/src/test/java/org/opengroup/osdu/dataset/provider/gcp/dms/DatasetDmsServiceMapImplTest.java

+/*
+ * Copyright 2021 Google LLC
+ * Copyright 2021 EPAM Systems, Inc
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.opengroup.osdu.dataset.provider.gcp.dms;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.Collections;
+import java.util.Map;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.mockito.InjectMocks;
+import org.mockito.Mock;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
+import org.opengroup.osdu.dataset.provider.gcp.config.GcpPropertiesConfig;
+import org.opengroup.osdu.dataset.provider.gcp.model.dataset.DmsServicePropertiesEntity;
+import org.opengroup.osdu.dataset.provider.gcp.repository.DmsServicePropertiesEntityRepository;
+
+@RunWith(MockitoJUnitRunner.class)
+public class DatasetDmsServiceMapImplTest {
+
+  private static final String DATASET_KIND = "example";
+
+  @Mock
+  private DmsServicePropertiesEntityRepository dmsServicePropertiesEntityRepository;
+
+  @Mock
+  private GcpPropertiesConfig gcpPropertiesConfig;
+
+  @InjectMocks
+  private DatasetDmsServiceMapImpl datasetDmsServiceMap;
+
+  @Before
+  public void setUp() {
+    DmsServicePropertiesEntity entity = new DmsServicePropertiesEntity();
+    entity.setDatasetKind(DATASET_KIND);
+    Iterable<DmsServicePropertiesEntity> entities = Collections.singletonList(entity);
+    Mockito.when(this.dmsServicePropertiesEntityRepository.findAll()).thenReturn(entities);
+  }
+
+  @Test
+  public void shouldGetResourceTypeToDmsServiceMap() {
+    Map<String, DmsServiceProperties> actual = this.datasetDmsServiceMap.getResourceTypeToDmsServiceMap();
+    assertTrue(actual.containsKey(DATASET_KIND));
+  }
+
+}

testing/dataset-test-gcp/src/test/java/org/opengroup/osdu/dataset/TestDataset.java

@@ -90,13 +90,13 @@ public class TestDataset extends Dataset {
 		//Step 2: Upload File
 		String fileName = "testFile.txt";
 		String fileContents = "Hello World!";
-		String unsignedUploadUrl = cloudStorageUtilGcp
+		String fileSource = cloudStorageUtilGcp
 			.uploadCloudFileUsingProvidedCredentials(fileName, datasetInstructions.getStorageLocation(),
 				fileContents);
-		uploadedCloudFileUnsignedUrls.add(unsignedUploadUrl);
+		uploadedCloudFileUnsignedUrls.add(getFileUnsignedUrl(fileSource));
 
 		//Step 3: Register File
-		String datasetRegistry = createDatasetRegistry(INPUT_DATASET_FILE_JSON, unsignedUploadUrl);
+		String datasetRegistry = createDatasetRegistry(INPUT_DATASET_FILE_JSON, fileSource);
 		String recordId = registerDataset(datasetRegistry);
 
 		//Step 4: Retrieve File and validate contents
@@ -201,9 +201,7 @@ public class TestDataset extends Dataset {
 		IntTestFileInstructionsItem fileInstructionsItem = objectMapper
 			.convertValue(deliveryItem.getRetrievalProperties(), IntTestFileInstructionsItem.class);
 
-		Assert.assertNotNull(fileInstructionsItem.getUnsignedUrl());
-		Assert.assertNotNull(fileInstructionsItem.getSignedUrl());