Commit 68ca97e3 authored by Riabokon Stanislav(EPAM)[GCP]'s avatar Riabokon Stanislav(EPAM)[GCP]
Browse files

Merge branch 'gcp/feature/file-collection-impl' into 'master'

Implement file collection for DataSet on Anthos env. (GONRG-4624)

See merge request !182
parents 3056c1f2 9a4e594a
Pipeline #104616 passed with stages
in 27 minutes and 42 seconds
......@@ -34,7 +34,6 @@ Must have:
| `EXPIRATION_DAYS` | ex `1` | expiration for signed urls & connection strings | no | |
| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
| `REDIS_GROUP_PORT` | ex `1111` | Redis port | no | https://console.cloud.google.com/memorystore/redis/instances |
| `osdu.dataset.config.useRestDms` | `true` OR `false` | Allows to configure *DMS REST APIs usage* | no | - |
| `DMS_API_BASE` | ex `http://localhost:8081/api/file/v2/files` | *Only for local usage.* Allows to override DMS service base url value from Datastore. | no | - |
These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
......@@ -48,6 +47,39 @@ Usage of spring profiles is preferred.
| `OQMDRIVER` | `rabbitmq` | Oqm driver mode that defines which message broker will be used | no | - |
| `SERVICE_TOKEN_PROVIDER` | `GCP` or `OPENID` |Service account token provider, `GCP` means use Google service account `OPEIND` means use OpenId provider like `Keycloak` | no | - |
## Datastore configuration
There must be a kind `DmsServiceProperties` in default namespace, with DMS configuration,
Example:
| name | apiKey | dmsServiceBaseUrl | isStagingLocationSupported | isStorageAllowed |
| --- | --- |---| --- | --- |
| `name=dataset--File.*` | | `https://osdu-anthos.osdu.club/api/file/v2/files` | `true` | `true` |
| `name=dataset--FileCollection.*` | | `https://osdu-anthos.osdu.club/api/file/v2/file-collections` | `true` | `true` |
You can use the `INSERT` script below to bootstrap the data with valid records:
```roomsql
INSERT INTO public."DmsServiceProperties"(id, data)
VALUES
('dataset--File.*',
'{
"apiKey": "",
"datasetKind": "dataset--File.*",
"isStorageAllowed": true,
"dmsServiceBaseUrl": "https://osdu-anthos.osdu.club/api/file/v2/files",
"isStagingLocationSupported": true
}'),
('dataset--FileCollection.*',
'{
"apiKey": "",
"datasetKind": "dataset--FileCollection.*",
"isStorageAllowed": true,
"dmsServiceBaseUrl": "https://osdu-anthos.osdu.club/api/file/v2/file-collections",
"isStagingLocationSupported": true
}');
```
### Properties set in Partition service:
Note that properties can be set in Partition as `sensitive` in that case in property `value` should be present **not value itself**, but **ENV variable name**.
......
......@@ -33,8 +33,7 @@ Must have:
| `EXPIRATION_DAYS` | ex `1` | expiration for signed urls & connection strings | no | |
| `REDIS_GROUP_HOST` | ex `127.0.0.1` | Redis host for groups | no | https://console.cloud.google.com/memorystore/redis/instances |
| `REDIS_GROUP_PORT` | ex `1111` | Redis port | no | https://console.cloud.google.com/memorystore/redis/instances |
| `osdu.dataset.config.useRestDms` | `true` OR `false` | Allows to configure *DMS REST APIs usage* | no | - |
| `DMS_API_BASE` | ex `http://localhost:8081/api/file/v2/files` | *Only for local usage.* Allows to override DMS service base url value from Datastore. | no | - |
| `DMS_API_BASE` | ex `http://localhost:8081/api/file/v2/files` | *Only for local usage.* Allows to override DMS service base url value from Datastore. | no | - |
These variables define service behavior, and are used to switch between `anthos` or `gcp` environments, their overriding and usage in mixed mode was not tested.
Usage of spring profiles is preferred.
......@@ -55,7 +54,7 @@ Example:
| name | apiKey | dmsServiceBaseUrl | isStagingLocationSupported | isStorageAllowed |
| --- | --- | --- | --- | --- |
| `name=dataset--File.*` | | `https://community.gcp.gnrg-osdu.projects.epam.com/api/file/v2/files` | `true` | `true` |
| `name=dataset--File.*` | | | `false` | `true` |
| `name=dataset--FileCollection.*` | | `https://community.gcp.gnrg-osdu.projects.epam.com/api/file/v2/file-collections` | `true` | `true` |
## GCS configuration
......
......@@ -56,7 +56,7 @@
<dependency>
<groupId>org.opengroup.osdu</groupId>
<artifactId>core-lib-gcp</artifactId>
<version>0.14.0</version>
<version>0.15.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>junit</groupId>
......@@ -88,6 +88,15 @@
<scope>test</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>4.9.2</version>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
......
......@@ -20,7 +20,6 @@ package org.opengroup.osdu.dataset.provider.gcp.di;
import lombok.RequiredArgsConstructor;
import org.opengroup.osdu.dataset.dms.IDmsFactory;
import org.opengroup.osdu.dataset.provider.gcp.dms.GcpDmsFactory;
import org.opengroup.osdu.dataset.provider.gcp.service.IFileService;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
......@@ -30,8 +29,6 @@ import org.springframework.stereotype.Component;
@RequiredArgsConstructor
public class DmsClientFactory extends AbstractFactoryBean<IDmsFactory> {
private final IFileService fileDmsService;
@Override
public Class<?> getObjectType() {
return IDmsFactory.class;
......@@ -39,6 +36,6 @@ public class DmsClientFactory extends AbstractFactoryBean<IDmsFactory> {
@Override
protected IDmsFactory createInstance() {
return new GcpDmsFactory(fileDmsService);
return new GcpDmsFactory();
}
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.dataset.provider.gcp.di;
public interface EnvironmentResolver {
String getProviderKey();
String getTransferProtocol();
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.dataset.provider.gcp.di;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
@Service
@ConditionalOnProperty(name = "obmDriver", havingValue = "gcs")
public class GcsEnvironmentResolverImpl implements EnvironmentResolver {
@Override
public String getProviderKey() {
return "GCP";
}
@Override
public String getTransferProtocol() {
return "gs://";
}
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.dataset.provider.gcp.di;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
@Service
@ConditionalOnProperty(name = "obmDriver", havingValue = "minio")
public class MinioEnvironmentResolverImpl implements EnvironmentResolver {
@Override
public String getProviderKey() {
return "ANTHOS";
}
@Override
public String getTransferProtocol() {
return "http://";
}
}
......@@ -24,20 +24,14 @@ import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
import org.opengroup.osdu.dataset.dms.IDmsFactory;
import org.opengroup.osdu.dataset.dms.IDmsProvider;
import org.opengroup.osdu.dataset.provider.gcp.model.dataset.GcpDmsServiceProperties;
import org.opengroup.osdu.dataset.provider.gcp.service.IFileService;
@RequiredArgsConstructor
public class GcpDmsFactory implements IDmsFactory {
private final IFileService fileDmsService;
private final IHttpClient httpClient = new HttpClient();
// TODO: rewrite DmsProvider injection into DmsProvider
@Override
public IDmsProvider create(DpsHeaders headers, DmsServiceProperties dmsServiceRoute) {
return new GcpDmsService(fileDmsService, (GcpDmsServiceProperties) dmsServiceRoute,
new GcpDmsRestService(dmsServiceRoute, httpClient, headers));
return new GcpDmsRestService(dmsServiceRoute, httpClient, headers);
}
}
......@@ -20,8 +20,12 @@ package org.opengroup.osdu.dataset.provider.gcp.dms;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.net.URISyntaxException;
import java.util.ArrayList;
import java.util.List;
import org.apache.http.HttpStatus;
import org.apache.http.client.utils.URIBuilder;
import org.opengroup.osdu.core.common.dms.model.DatasetRetrievalProperties;
import org.opengroup.osdu.core.common.dms.model.RetrievalInstructionsResponse;
import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.IHttpClient;
......@@ -29,6 +33,9 @@ import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.dataset.dms.DmsRestService;
import org.opengroup.osdu.dataset.dms.DmsServiceProperties;
import org.opengroup.osdu.dataset.model.request.GetDatasetRegistryRequest;
import org.opengroup.osdu.dataset.model.response.DatasetRetrievalDeliveryItem;
import org.opengroup.osdu.dataset.model.response.GetDatasetRetrievalInstructionsResponse;
import org.opengroup.osdu.dataset.model.response.GetDatasetStorageInstructionsResponse;
public class GcpDmsRestService extends DmsRestService {
......@@ -61,6 +68,23 @@ public class GcpDmsRestService extends DmsRestService {
}
}
@Override
public GetDatasetRetrievalInstructionsResponse getDatasetRetrievalInstructions(GetDatasetRegistryRequest request) {
RetrievalInstructionsResponse retrievalInstructions = super.getRetrievalInstructions(request);
String providerKey = retrievalInstructions.getProviderKey();
List<DatasetRetrievalDeliveryItem> datasetRetrievalDeliveryItemList = new ArrayList<>();
for (DatasetRetrievalProperties properties : retrievalInstructions.getDatasets()) {
DatasetRetrievalDeliveryItem retrievalDeliveryItem = new DatasetRetrievalDeliveryItem(
properties.getDatasetRegistryId(), properties.getRetrievalProperties(),
providerKey);
datasetRetrievalDeliveryItemList.add(retrievalDeliveryItem);
}
return new GetDatasetRetrievalInstructionsResponse(datasetRetrievalDeliveryItemList);
}
private String createUrl(String path) {
try {
URIBuilder uriBuilder = new URIBuilder(dmsServiceProperties.getDmsServiceBaseUrl());
......
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.dataset.provider.gcp.dms;
import java.util.ArrayList;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.dms.model.CopyDmsRequest;
import org.opengroup.osdu.core.common.dms.model.CopyDmsResponse;
import org.opengroup.osdu.core.common.dms.model.DatasetRetrievalProperties;
import org.opengroup.osdu.core.common.dms.model.RetrievalInstructionsResponse;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.dataset.dms.DmsException;
import org.opengroup.osdu.dataset.dms.IDmsProvider;
import org.opengroup.osdu.dataset.model.request.GetDatasetRegistryRequest;
import org.opengroup.osdu.dataset.model.response.DatasetRetrievalDeliveryItem;
import org.opengroup.osdu.dataset.model.response.GetDatasetRetrievalInstructionsResponse;
import org.opengroup.osdu.dataset.model.response.GetDatasetStorageInstructionsResponse;
import org.opengroup.osdu.dataset.provider.gcp.model.dataset.GcpDmsServiceProperties;
import org.opengroup.osdu.dataset.provider.gcp.service.IFileService;
@RequiredArgsConstructor
public class GcpDmsService implements IDmsProvider {
private final IFileService fileDmsService;
private final GcpDmsServiceProperties dmsServiceProperties;
private final IDmsProvider dmsRestService;
// TODO: osdu.dataset.config.useRestDms property ignored in current implementation
@Override
public GetDatasetStorageInstructionsResponse getStorageInstructions() throws DmsException {
switch (dmsServiceProperties.getDataSetType()) {
case FILE:
return dmsRestService.getStorageInstructions();
case FILE_COLLECTION:
return fileDmsService.getCollectionUploadInstructions();
default:
throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request",
"Invalid dataset provided");
}
}
@Override
public GetDatasetRetrievalInstructionsResponse getDatasetRetrievalInstructions(
GetDatasetRegistryRequest request)
throws DmsException {
switch (dmsServiceProperties.getDataSetType()) {
case FILE:
RetrievalInstructionsResponse retrievalInstructions = dmsRestService.getRetrievalInstructions(request);
String providerKey = retrievalInstructions.getProviderKey();
List<DatasetRetrievalDeliveryItem> datasetRetrievalDeliveryItemList = new ArrayList<>();
for (DatasetRetrievalProperties properties : retrievalInstructions.getDatasets()) {
DatasetRetrievalDeliveryItem retrievalDeliveryItem = new DatasetRetrievalDeliveryItem(
properties.getDatasetRegistryId(), properties.getRetrievalProperties(),
providerKey);
datasetRetrievalDeliveryItemList.add(retrievalDeliveryItem);
}
return new GetDatasetRetrievalInstructionsResponse(datasetRetrievalDeliveryItemList);
case FILE_COLLECTION:
return fileDmsService.getCollectionRetrievalInstructions(request);
default:
throw new AppException(HttpStatus.SC_BAD_REQUEST, "Bad request",
"Invalid dataset provided");
}
}
@Override
public RetrievalInstructionsResponse getRetrievalInstructions(GetDatasetRegistryRequest request)
throws DmsException {
return dmsRestService.getRetrievalInstructions(request);
}
@Override
public List<CopyDmsResponse> copyDmsToPersistentStorage(CopyDmsRequest copyDmsRequest)
throws DmsException {
return dmsRestService.copyDmsToPersistentStorage(copyDmsRequest);
}
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.dataset.provider.gcp.model;
import com.fasterxml.jackson.annotation.JsonProperty;
import java.time.Instant;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
public class FileCollectionInstructionsItem {
@JsonProperty("unsignedUrl")
private String unsignedUrl;
@JsonProperty("connectionString")
private String connectionString;
@JsonProperty("createdAt")
private Instant createdAt;
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.dataset.provider.gcp.model;
public class GcsRole {
public static final String STORAGE_OBJECT_VIEWER = "storage.objectViewer";
public static final String STORAGE_OBJECT_ADMIN = "storage.objectAdmin";
private GcsRole() {
}
}
/*
* Copyright 2021 Google LLC
* Copyright 2021 EPAM Systems, Inc
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.opengroup.osdu.dataset.provider.gcp.service;
import com.fasterxml.jackson.databind.ObjectMapper;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpStatus;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyParsingException;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.storage.MultiRecordInfo;
import org.opengroup.osdu.core.common.model.storage.Record;
import org.opengroup.osdu.core.common.model.storage.StorageException;
import org.opengroup.osdu.core.common.storage.IStorageFactory;
import org.opengroup.osdu.core.common.storage.IStorageService;
import org.opengroup.osdu.dataset.model.request.GetDatasetRegistryRequest;
import org.opengroup.osdu.dataset.model.request.StorageExceptionResponse;
import org.opengroup.osdu.dataset.model.response.DatasetRetrievalDeliveryItem;
import org.opengroup.osdu.dataset.model.response.GetDatasetRetrievalInstructionsResponse;
import org.opengroup.osdu.dataset.model.response.GetDatasetStorageInstructionsResponse;
import org.opengroup.osdu.dataset.provider.gcp.config.GcpConfigProperties;
import org.opengroup.osdu.dataset.provider.gcp.di.EnvironmentResolver;
import org.opengroup.osdu.dataset.provider.gcp.model.FileCollectionInstructionsItem;
import org.opengroup.osdu.dataset.provider.gcp.model.FileInstructionsItem;
import org.opengroup.osdu.dataset.provider.gcp.model.dataset.GcpDatasetRetrievalDeliveryItem;
import org.opengroup.osdu.dataset.provider.gcp.model.dataset.GcpGetDatasetStorageInstructionsResponse;
import org.opengroup.osdu.dataset.provider.gcp.service.instructions.interfaces.IFileCollectionStorageService;
import org.opengroup.osdu.dataset.provider.gcp.service.instructions.interfaces.IFileStorageService;
import org.springframework.stereotype.Service;
@Slf4j
@RequiredArgsConstructor
@Service
public class FileServiceImpl implements IFileService {
private final HttpResponseBodyMapper bodyMapper;
private final ObjectMapper objectMapper;
private final IFileStorageService fileStorageService;
private final IFileCollectionStorageService collectionStorageService;
private final IStorageFactory storageFactory;
private final GcpConfigProperties propertiesConfig;
private final DpsHeaders headers;
private final EnvironmentResolver providerKeyResolver;
@Override
public GetDatasetStorageInstructionsResponse getFileUploadInstructions() {
return new GcpGetDatasetStorageInstructionsResponse(fileStorageService.getFileUploadItem(),
objectMapper, providerKeyResolver.getProviderKey());
}
@Override
public GetDatasetStorageInstructionsResponse getCollectionUploadInstructions() {
return new GcpGetDatasetStorageInstructionsResponse(collectionStorageService.getCollectionUploadItem(),
objectMapper, providerKeyResolver.getProviderKey());
}
@Override
public GetDatasetRetrievalInstructionsResponse getFileRetrievalInstructions(
GetDatasetRegistryRequest getDatasetRegistryRequest) {
MultiRecordInfo getRecordsResponse = getMultiRecordInfo(getDatasetRegistryRequest);
List<DatasetRetrievalDeliveryItem> delivery = getFileDelivery(getRecordsResponse.getRecords());
return new GetDatasetRetrievalInstructionsResponse(delivery);
}
@Override
public GetDatasetRetrievalInstructionsResponse getCollectionRetrievalInstructions(
GetDatasetRegistryRequest getDatasetRegistryRequest) {
MultiRecordInfo getRecordsResponse = getMultiRecordInfo(getDatasetRegistryRequest);
List<DatasetRetrievalDeliveryItem> delivery = getFileCollectionDelivery(getRecordsResponse.getRecords());
return new GetDatasetRetrievalInstructionsResponse(delivery);
}
private MultiRecordInfo getMultiRecordInfo(GetDatasetRegistryRequest getDatasetRegistryRequest) {
IStorageService storageService = this.storageFactory.create(headers);
MultiRecordInfo getRecordsResponse = null;
try {
getRecordsResponse = storageService.getRecords(getDatasetRegistryRequest.datasetRegistryIds);
} catch (StorageException e) {
try {
StorageExceptionResponse body = bodyMapper
.parseBody(e.getHttpResponse(), StorageExceptionResponse.class);
throw new AppException(body.getCode(), "Storage Service: " + body.getReason(), body.getMessage());
} catch (HttpResponseBodyParsingException e1) {
throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR,