Commit 5bb6f9e5 authored by Anastasiia Gelmut's avatar Anastasiia Gelmut Committed by Riabokon Stanislav(EPAM)[GCP]
Browse files

Remap Dataset To Use File Service (GONRG-2918)

parent da658b8f
......@@ -67,3 +67,5 @@ osdu-gcp-test:
PROVIDER_KEY: GCP
KIND_SUBTYPE: DatasetTest
LEGAL_TAG: datatest-int-tag
LEGAL_HOST: $OSDU_GCP_LEGALTAG_API/
GCP_STORAGE_PERSISTENT_AREA: persisten-area
This diff is collapsed.
......@@ -19,6 +19,8 @@ package org.opengroup.osdu.dataset.dms;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.net.URISyntaxException;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.apache.http.HttpStatus;
import org.apache.http.client.utils.URIBuilder;
......@@ -33,8 +35,6 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.dataset.model.request.GetDatasetRegistryRequest;
import org.opengroup.osdu.dataset.model.response.GetDatasetRetrievalInstructionsResponse;
import org.opengroup.osdu.dataset.model.response.GetDatasetStorageInstructionsResponse;
import java.net.URISyntaxException;
import java.util.List;
@RequiredArgsConstructor
public class DmsRestService implements IDmsProvider {
......
......@@ -25,7 +25,9 @@ In order to run the service locally or remotely, you will need to have the follo
| `REDIS_GROUP_PORT` | ex `1111` | Redis port | no | https://console.cloud.google.com/memorystore/redis/instances |
| `GOOGLE_AUDIENCES` | ex `*****.apps.googleusercontent.com` | Client ID for getting access to cloud resources | yes | https://console.cloud.google.com/apis/credentials |
| `PARTITION_API` | ex `http://localhost:8081/api/partition/v1` | Partition service endpoint | no | - |
| `osdu.dataset.config.useRestDms` | `true` OR `false` | Allows to configure DMS REST APIs usage | no | - |
| `spring.cloud.gcp.datastore.namespace` | ex `namespace` | Allows to configure Datastore namespace for DMS service properties storage | no | - |
| `DMS_API_BASE` | ex `http://localhost:8081/api/file/v2/files` | Only for local usage. Allows to override DMS service base url value from Datastore. | no | - |
### Run Locally
Check that maven is installed:
......@@ -119,13 +121,14 @@ cd provider/dataset-gcp && mvn spring-boot:run
| `KIND_SUBTYPE` | `DatasetTest` | Kind subtype that will be used in int tests, schema creation automated , result kind will be `TENANT_NAME::wks-test:dataset--FileCollection.KIND_SUBTYPE:1.0.0`| no | - |
| `LEGAL_TAG` | `public-usa-dataset-1` | Legal tag name, if tag with that name doesn't exist then it will be created during preparing step | no | - |
| `GCLOUD_PROJECT` | `osdu-cicd-epam` | Project id | no | - |
| `GCP_STORAGE_PERSISTENT_AREA` | ex `persistent-area` | persistent area bucket(will be concatenated with project id ex `osdu-cicd-epam-persistent-area` | no | output of infrastructure deployment |
| `LEGAL_HOST` | ex `https://os-legal-jvmvia5dea-uc.a.run.app/api/legal/v1/` | Legal API endpoint | no | output of infrastructure deployment |
**Entitlements configuration for integration accounts**
| INTEGRATION_TESTER |
| --- |
| users<br/>service.entitlements.user<br/>service.storage.admin<br/>service.legal.user<br/>service.search.user<br/>service.delivery.viewer |
| users<br/>service.entitlements.user<br/>service.storage.admin<br/>service.legal.user<br/>service.search.user<br/>service.delivery.viewer<br/>service.dataset.viewers<br/>service.dataset.editors |
**Cloud roles configuration for integration accounts**
......
......@@ -51,6 +51,11 @@
<artifactId>core-lib-gcp</artifactId>
<version>0.9.0</version>
</dependency>
<dependency>
<groupId>org.springframework.cloud</groupId>
<artifactId>spring-cloud-gcp-starter-data-datastore</artifactId>
<version>1.2.5.RELEASE</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
......
......@@ -23,6 +23,7 @@ import org.opengroup.osdu.dataset.di.DmsClientFactory;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
import org.springframework.cloud.gcp.data.datastore.repository.config.EnableDatastoreRepositories;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.FilterType;
......@@ -31,6 +32,7 @@ import org.springframework.context.annotation.FilterType;
@ComponentScan(value = {"org.opengroup.osdu"}, excludeFilters = {
@ComponentScan.Filter(type = FilterType.ASSIGNABLE_TYPE, value = {DmsClientFactory.class,
DatasetApplication.class, IStorageFactory.class})})
@EnableDatastoreRepositories
public class DatasetApplicationGCP {
public static void main(String[] args) {
......
......@@ -36,4 +36,6 @@ public class GcpPropertiesConfig {
private String redisGroupHost;
private Integer redisGroupPort;
private String dmsApiBase;
}
......@@ -20,9 +20,13 @@ package org.opengroup.osdu.dataset.provider.gcp.dms;
import java.util.HashMap;
import java.util.Map;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
import org.opengroup.osdu.dataset.provider.gcp.config.GcpPropertiesConfig;
import org.opengroup.osdu.dataset.provider.gcp.model.dataset.DataSetType;
import org.opengroup.osdu.dataset.provider.gcp.model.dataset.DmsServicePropertiesEntity;
import org.opengroup.osdu.dataset.provider.gcp.model.dataset.GcpDmsServiceProperties;
import org.opengroup.osdu.dataset.provider.gcp.repository.DmsServicePropertiesEntityRepository;
import org.opengroup.osdu.dataset.provider.interfaces.IDatasetDmsServiceMap;
import org.springframework.stereotype.Service;
......@@ -30,20 +34,34 @@ import org.springframework.stereotype.Service;
@RequiredArgsConstructor
public class DatasetDmsServiceMapImpl implements IDatasetDmsServiceMap {
private final Map<String, DmsServiceProperties> resourceTypeToDmsServiceMap = new HashMap<>();
private final DmsServicePropertiesEntityRepository dmsServicePropertiesEntityRepository;
private final GcpPropertiesConfig gcpPropertiesConfig;
@Override
public Map<String, DmsServiceProperties> getResourceTypeToDmsServiceMap() {
resourceTypeToDmsServiceMap.put(
"dataset--File.*",
new GcpDmsServiceProperties(DataSetType.FILE)
);
@Override
public Map<String, DmsServiceProperties> getResourceTypeToDmsServiceMap() {
Iterable<DmsServicePropertiesEntity> properties = this.dmsServicePropertiesEntityRepository.findAll();
resourceTypeToDmsServiceMap.put(
"dataset--FileCollection.*",
new GcpDmsServiceProperties(DataSetType.FILE_COLLECTION)
);
Map<String, DmsServiceProperties> resourceTypeToDmsServiceMap = new HashMap<>();
for (DmsServicePropertiesEntity entity : properties) {
String dmsApiBase = this.gcpPropertiesConfig.getDmsApiBase();
if (StringUtils.isEmpty(dmsApiBase)) {
dmsApiBase = entity.getDmsServiceBaseUrl();
}
DmsServiceProperties dmsServiceProperties = new DmsServiceProperties(dmsApiBase, entity.isStorageAllowed(),
entity.getApiKey(), entity.isStagingLocationSupported());
if (StringUtils.containsIgnoreCase(entity.getDatasetKind(), "collection")) {
resourceTypeToDmsServiceMap.put(entity.getDatasetKind(), new GcpDmsServiceProperties(
DataSetType.FILE_COLLECTION, dmsServiceProperties));
} else {
resourceTypeToDmsServiceMap.put(entity.getDatasetKind(), new GcpDmsServiceProperties(
DataSetType.FILE, dmsServiceProperties));
}
}
return resourceTypeToDmsServiceMap;
}
return resourceTypeToDmsServiceMap;
}
}
......@@ -18,7 +18,10 @@
package org.opengroup.osdu.dataset.provider.gcp.dms;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.core.common.http.HttpClient;
import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.dataset.dms.DmsRestService;
import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
import org.opengroup.osdu.dataset.dms.IDmsFactory;
import org.opengroup.osdu.dataset.dms.IDmsProvider;
......@@ -28,10 +31,13 @@ import org.opengroup.osdu.dataset.provider.gcp.service.IFileService;
@RequiredArgsConstructor
public class GcpDmsFactory implements IDmsFactory {
private final IFileService fileDmsService;
private final IFileService fileDmsService;
@Override
public IDmsProvider create(DpsHeaders headers, DmsServiceProperties dmsServiceRoute) {
return new GcpDmsService(fileDmsService, (GcpDmsServiceProperties) dmsServiceRoute);
}
private IHttpClient httpClient = new HttpClient();
@Override
public IDmsProvider create(DpsHeaders headers, DmsServiceProperties dmsServiceRoute) {
return new GcpDmsService(fileDmsService, (GcpDmsServiceProperties) dmsServiceRoute,
new GcpDmsRestService(dmsServiceRoute, httpClient, headers));
}
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.dataset.provider.gcp.dms;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.net.URISyntaxException;
import org.apache.http.HttpStatus;
import org.apache.http.client.utils.URIBuilder;
import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.dataset.dms.DmsRestService;
import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
import org.opengroup.osdu.dataset.model.response.GetDatasetStorageInstructionsResponse;
public class GcpDmsRestService extends DmsRestService {
private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private final DmsServiceProperties dmsServiceProperties;
private final IHttpClient httpClient;
private final DpsHeaders headers;
public GcpDmsRestService(DmsServiceProperties dmsServiceProperties,
IHttpClient httpClient,
DpsHeaders headers) {
super(dmsServiceProperties, httpClient, headers);
this.dmsServiceProperties = dmsServiceProperties;
this.httpClient = httpClient;
this.headers = headers;
}
@Override
public GetDatasetStorageInstructionsResponse getStorageInstructions() {
String url = this.createUrl("/storageInstructions");
HttpResponse result = this.httpClient
.send(HttpRequest.post("").url(url).headers(this.headers.getHeaders()).build());
try {
return OBJECT_MAPPER.readValue(result.getBody(), GetDatasetStorageInstructionsResponse.class);
} catch (JsonProcessingException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Internal Server Error",
e.getMessage(), e);
}
}
private String createUrl(String path) {
try {
URIBuilder uriBuilder = new URIBuilder(dmsServiceProperties.getDmsServiceBaseUrl());
uriBuilder.setPath(uriBuilder.getPath() + path);
return uriBuilder.build().normalize().toString();
} catch (URISyntaxException e) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, "Invalid URL", e.getMessage(), e);
}
}
}
......@@ -17,12 +17,19 @@
package org.opengroup.osdu.dataset.provider.gcp.dms;
import java.util.ArrayList;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.dms.model.CopyDmsRequest;
import org.opengroup.osdu.core.common.dms.model.CopyDmsResponse;
import org.opengroup.osdu.core.common.dms.model.DatasetRetrievalProperties;
import org.opengroup.osdu.core.common.dms.model.RetrievalInstructionsResponse;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.dataset.dms.DmsException;
import org.opengroup.osdu.dataset.dms.IDmsProvider;
import org.opengroup.osdu.dataset.model.request.GetDatasetRegistryRequest;
import org.opengroup.osdu.dataset.model.response.DatasetRetrievalDeliveryItem;
import org.opengroup.osdu.dataset.model.response.GetDatasetRetrievalInstructionsResponse;
import org.opengroup.osdu.dataset.model.response.GetDatasetStorageInstructionsResponse;
import org.opengroup.osdu.dataset.provider.gcp.model.dataset.GcpDmsServiceProperties;
......@@ -31,32 +38,61 @@ import org.opengroup.osdu.dataset.provider.gcp.service.IFileService;
@RequiredArgsConstructor
public class GcpDmsService implements IDmsProvider {
private final IFileService fileDmsService;
private final GcpDmsServiceProperties dmsServiceProperties;
@Override
public GetDatasetStorageInstructionsResponse getStorageInstructions() throws DmsException {
switch (dmsServiceProperties.getDataSetType()) {
case FILE:
return fileDmsService.getFileUploadInstructions();
case FILE_COLLECTION:
return fileDmsService.getCollectionUploadInstructions();
default:
throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "Invalid dataset provided");
}
}
@Override
public GetDatasetRetrievalInstructionsResponse getDatasetRetrievalInstructions(GetDatasetRegistryRequest request)
throws DmsException {
switch (dmsServiceProperties.getDataSetType()) {
case FILE:
return fileDmsService.getFileRetrievalInstructions(request);
case FILE_COLLECTION:
return fileDmsService.getCollectionRetrievalInstructions(request);
default:
throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request", "Invalid dataset provided");
}
}
private final IFileService fileDmsService;
private final GcpDmsServiceProperties dmsServiceProperties;
private final IDmsProvider dmsRestService;
@Override
public GetDatasetStorageInstructionsResponse getStorageInstructions() throws DmsException {
switch (dmsServiceProperties.getDataSetType()) {
case FILE:
return dmsRestService.getStorageInstructions();
case FILE_COLLECTION:
return fileDmsService.getCollectionUploadInstructions();
default:
throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request",
"Invalid dataset provided");
}
}
@Override
public GetDatasetRetrievalInstructionsResponse getDatasetRetrievalInstructions(
GetDatasetRegistryRequest request)
throws DmsException {
switch (dmsServiceProperties.getDataSetType()) {
case FILE:
RetrievalInstructionsResponse retrievalInstructions = dmsRestService.getRetrievalInstructions(request);
String providerKey = retrievalInstructions.getProviderKey();
List<DatasetRetrievalDeliveryItem> datasetRetrievalDeliveryItemList = new ArrayList<>();
for (DatasetRetrievalProperties properties : retrievalInstructions.getDatasets()) {
DatasetRetrievalDeliveryItem retrievalDeliveryItem = new DatasetRetrievalDeliveryItem(
properties.getDatasetRegistryId(), properties.getRetrievalProperties(),
providerKey);
datasetRetrievalDeliveryItemList.add(retrievalDeliveryItem);
}
return new GetDatasetRetrievalInstructionsResponse(datasetRetrievalDeliveryItemList);
case FILE_COLLECTION:
return fileDmsService.getCollectionRetrievalInstructions(request);
default:
throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request",
"Invalid dataset provided");
}
}
@Override
public RetrievalInstructionsResponse getRetrievalInstructions(GetDatasetRegistryRequest request)
throws DmsException {
return dmsRestService.getRetrievalInstructions(request);
}
@Override
public List<CopyDmsResponse> copyDmsToPersistentStorage(CopyDmsRequest copyDmsRequest)
throws DmsException {
return dmsRestService.copyDmsToPersistentStorage(copyDmsRequest);
}
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.dataset.provider.gcp.model.dataset;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.cloud.gcp.data.datastore.core.mapping.Entity;
import org.springframework.data.annotation.Id;
@Entity(name = "DmsServiceProperties")
@Data
@NoArgsConstructor
@AllArgsConstructor
public class DmsServicePropertiesEntity {
@Id
private String datasetKind;
private String dmsServiceBaseUrl;
private boolean isStorageAllowed;
private String apiKey;
private boolean isStagingLocationSupported;
}
......@@ -27,5 +27,11 @@ import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
@AllArgsConstructor
public class GcpDmsServiceProperties extends DmsServiceProperties {
private DataSetType dataSetType;
private DataSetType dataSetType;
public GcpDmsServiceProperties(DataSetType dataSetType, DmsServiceProperties dmsServiceProperties) {
super(dmsServiceProperties.getDmsServiceBaseUrl(), dmsServiceProperties.isAllowStorage(),
dmsServiceProperties.getApiKey(), dmsServiceProperties.isStagingLocationSupported());
this.dataSetType = dataSetType;
}
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.dataset.provider.gcp.repository;
import org.opengroup.osdu.dataset.provider.gcp.model.dataset.DmsServicePropertiesEntity;
import org.springframework.cloud.gcp.data.datastore.repository.DatastoreRepository;
import org.springframework.stereotype.Repository;
@Repository
public interface DmsServicePropertiesEntityRepository extends
DatastoreRepository<DmsServicePropertiesEntity, String> {
}
......@@ -36,3 +36,9 @@ LOG_PREFIX=${log.prefix}
google.audiences=123.apps.googleusercontent.com
partition.api=http://localhost:8081/api/partition/v1
DMS_API_BASE=
osdu.dataset.config.useRestDms=false
spring.cloud.gcp.datastore.namespace=
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.dataset.provider.gcp.dms;
import static org.junit.Assert.assertTrue;
import java.util.Collections;
import java.util.Map;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.runners.MockitoJUnitRunner;
import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
import org.opengroup.osdu.dataset.provider.gcp.config.GcpPropertiesConfig;
import org.opengroup.osdu.dataset.provider.gcp.model.dataset.DmsServicePropertiesEntity;
import org.opengroup.osdu.dataset.provider.gcp.repository.DmsServicePropertiesEntityRepository;
@RunWith(MockitoJUnitRunner.class)
public class DatasetDmsServiceMapImplTest {
private static final String DATASET_KIND = "example";
@Mock
private DmsServicePropertiesEntityRepository dmsServicePropertiesEntityRepository;
@Mock
private GcpPropertiesConfig gcpPropertiesConfig;
@InjectMocks
private DatasetDmsServiceMapImpl datasetDmsServiceMap;
@Before
public void setUp() {
DmsServicePropertiesEntity entity = new DmsServicePropertiesEntity();
entity.setDatasetKind(DATASET_KIND);
Iterable<DmsServicePropertiesEntity> entities = Collections.singletonList(entity);
Mockito.when(this.dmsServicePropertiesEntityRepository.findAll()).thenReturn(entities);
}
@Test
public void shouldGetResourceTypeToDmsServiceMap() {
Map<String, DmsServiceProperties> actual = this.datasetDmsServiceMap.getResourceTypeToDmsServiceMap();
assertTrue(actual.containsKey(DATASET_KIND));
}
}
......@@ -90,13 +90,13 @@ public class TestDataset extends Dataset {
//Step 2: Upload File
String fileName = "testFile.txt";
String fileContents = "Hello World!";
String unsignedUploadUrl = cloudStorageUtilGcp
String fileSource = cloudStorageUtilGcp
.uploadCloudFileUsingProvidedCredentials(fileName, datasetInstructions.getStorageLocation(),
fileContents);
uploadedCloudFileUnsignedUrls.add(unsignedUploadUrl);
uploadedCloudFileUnsignedUrls.add(getFileUnsignedUrl(fileSource));
//Step 3: Register File
String datasetRegistry = createDatasetRegistry(INPUT_DATASET_FILE_JSON, unsignedUploadUrl);
String datasetRegistry = createDatasetRegistry(INPUT_DATASET_FILE_JSON, fileSource);
String recordId = registerDataset(datasetRegistry);
//Step 4: Retrieve File and validate contents
......@@ -201,9 +201,7 @@ public class TestDataset extends Dataset {
IntTestFileInstructionsItem fileInstructionsItem = objectMapper
.convertValue(deliveryItem.getRetrievalProperties(), IntTestFileInstructionsItem.class);
Assert.assertNotNull(fileInstructionsItem.getUnsignedUrl());
Assert.assertNotNull(fileInstructionsItem.getSignedUrl());
Assert.assertNotNull(fileInstructionsItem.getCreatedAt());
Assert.assertNotNull(fileInstructionsItem);
}