Commit a7740699 authored by Matt Wise's avatar Matt Wise
Browse files

R3 Data Definitions Changes

commit 7697e607 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Tue Feb 02 2021 13:47:07 GMT-0600 (Central Standard Time) 

    update comment on dataset kind regex


commit 561cde42 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Tue Feb 02 2021 11:58:29 GMT-0600 (Central Standard Time) 

    add provider key to swagger docs


commit 36166ff7 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Tue Feb 02 2021 11:49:02 GMT-0600 (Central Standard Time) 

    update core-lib-aws


commit b22e2104 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Tue Feb 02 2021 11:30:57 GMT-0600 (Central Standard Time) 

    update swagger docs. Remove legacy docs


commit df702cdc 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Tue Feb 02 2021 11:22:24 GMT-0600 (Central Standard Time) 

    Updates to DMS API to support R3 data defs. Update int tests


commit cdf92e61 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Tue Feb 02 2021 09:09:59 GMT-0600 (Central Standard Time) 

    Work on supporting R3 data def for DatasetRegistryAPI/Service


commit 204fd922 
Author: Matt Wise <wsmatth@amazon.com> 
Date: Fri Jan 29 2021 08:43:08 GMT-0600 (Central Standard Time) 

    Add schema service client
parent 9ac42214
Pipeline #24503 failed with stages
in 20 minutes and 6 seconds
......@@ -54,9 +54,9 @@ public class DatasetDmsApi {
@GetMapping("/getStorageInstructions")
@PreAuthorize("@authorizationFilter.hasRole('" + DeliveryRole.VIEWER + "')")
public ResponseEntity<GetDatasetStorageInstructionsResponse> getStorageInstructions(
@RequestParam(value = "resourceType") String resourceType) {
@RequestParam(value = "kindSubType") String kindSubType) {
GetDatasetStorageInstructionsResponse response = this.datasetDmsService.getStorageInstructions(resourceType);
GetDatasetStorageInstructionsResponse response = this.datasetDmsService.getStorageInstructions(kindSubType);
return new ResponseEntity<GetDatasetStorageInstructionsResponse>(response, HttpStatus.OK);
}
......
......@@ -15,9 +15,12 @@
package org.opengroup.osdu.dataset.di;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
......@@ -25,6 +28,9 @@ import org.springframework.stereotype.Component;
@Component
public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
private final ObjectMapper objectMapper = new ObjectMapper();
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
@Value("${AUTHORIZE_API}")
private String AUTHORIZE_API;
......@@ -38,7 +44,8 @@ public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlements
.builder()
.rootUrl(AUTHORIZE_API)
.apiKey(AUTHORIZE_API_KEY)
.build());
.build(),
bodyMapper);
}
@Override
......
......@@ -13,22 +13,38 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//TODO: Move to os-core-common
package org.opengroup.osdu.dataset.di;
package org.opengroup.osdu.dataset.storage;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.opengroup.osdu.core.common.model.http.DpsException;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.dataset.schema.ISchemaFactory;
import org.opengroup.osdu.dataset.schema.SchemaAPIConfig;
import org.opengroup.osdu.dataset.schema.SchemaFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
@Data
@EqualsAndHashCode(callSuper = false)
public class StorageException extends DpsException {
@Component
public class SchemaClientFactory extends AbstractFactoryBean<ISchemaFactory> {
private static final long serialVersionUID = 9094949225576291097L;
private final ObjectMapper objectMapper = new ObjectMapper();
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
public StorageException(String message, HttpResponse httpResponse) {
super(message, httpResponse);
}
@Value("${SCHEMA_API}")
private String SCHEMA_API;
@Override
public Class<?> getObjectType() {
return ISchemaFactory.class;
}
@Override
protected ISchemaFactory createInstance() throws Exception {
return new SchemaFactory(SchemaAPIConfig
.builder()
.rootUrl(SCHEMA_API)
.build(),
bodyMapper);
}
}
\ No newline at end of file
......@@ -15,12 +15,12 @@
package org.opengroup.osdu.dataset.di;
import org.opengroup.osdu.core.common.legal.ILegalFactory;
import org.opengroup.osdu.core.common.legal.LegalAPIConfig;
import org.opengroup.osdu.core.common.legal.LegalFactory;
import org.opengroup.osdu.dataset.storage.IStorageFactory;
import org.opengroup.osdu.dataset.storage.StorageAPIConfig;
import org.opengroup.osdu.dataset.storage.StorageFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.storage.IStorageFactory;
import org.opengroup.osdu.core.common.storage.StorageAPIConfig;
import org.opengroup.osdu.core.common.storage.StorageFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
......@@ -28,6 +28,9 @@ import org.springframework.stereotype.Component;
@Component
public class StorageClientFactory extends AbstractFactoryBean<IStorageFactory> {
private final ObjectMapper objectMapper = new ObjectMapper();
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
@Value("${STORAGE_API}")
private String STORAGE_API;
......@@ -40,7 +43,8 @@ public class StorageClientFactory extends AbstractFactoryBean<IStorageFactory> {
protected IStorageFactory createInstance() throws Exception {
return new StorageFactory(StorageAPIConfig
.builder()
.rootUrl(STORAGE_API)
.build());
.rootUrl(STORAGE_API)
.build(),
bodyMapper);
}
}
\ No newline at end of file
......@@ -17,12 +17,16 @@
package org.opengroup.osdu.dataset.dms;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.JsonSyntaxException;
import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyParsingException;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.dataset.model.request.GetDatasetRegistryRequest;
......@@ -31,17 +35,19 @@ import org.opengroup.osdu.dataset.model.response.GetDatasetStorageInstructionsRe
import org.springframework.http.HttpStatus;
public class DmsService implements IDmsProvider {
private String dmsServiceUrl;
private DmsServiceProperties dmsServiceProperties;
private final IHttpClient httpClient;
private final DpsHeaders headers;
private final ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
public DmsService(DmsServiceProperties dmsServiceProperties, IHttpClient httpClient, DpsHeaders headers) {
this.dmsServiceProperties = dmsServiceProperties;
this.dmsServiceUrl = dmsServiceProperties.getDmsServiceBaseUrl();
this.dmsServiceUrl = dmsServiceProperties.getDmsServiceBaseUrl();
this.httpClient = httpClient;
this.headers = headers;
......@@ -61,7 +67,8 @@ public class DmsService implements IDmsProvider {
}
@Override
public GetDatasetRetrievalInstructionsResponse getDatasetRetrievalInstructions(GetDatasetRegistryRequest datasetRegistryRequest) throws DmsException {
public GetDatasetRetrievalInstructionsResponse getDatasetRetrievalInstructions(
GetDatasetRegistryRequest datasetRegistryRequest) throws DmsException {
String url = this.createUrl("/getRetrievalInstructions");
HttpResponse result = this.httpClient
......@@ -69,7 +76,6 @@ public class DmsService implements IDmsProvider {
return this.getResult(result, GetDatasetRetrievalInstructionsResponse.class);
}
private String createUrl(String requestPathAndQuery) {
return StringUtils.join(this.dmsServiceUrl, requestPathAndQuery);
}
......@@ -77,8 +83,8 @@ public class DmsService implements IDmsProvider {
private <T> T getResult(HttpResponse result, Class<T> type) throws DmsException {
if (result.isSuccessCode()) {
try {
return result.parseBody(type);
} catch (JsonSyntaxException e) {
return bodyMapper.parseBody(result, type);
} catch (JsonSyntaxException | HttpResponseBodyParsingException e) {
throw new DmsException("Error parsing response. Check the inner HttpResponse for more info.",
result);
}
......
// Copyright © 2020 Amazon Web Services
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
......@@ -13,19 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//TODO: move this to os-core-common
package org.opengroup.osdu.dataset.storage;
import java.util.List;
import org.opengroup.osdu.core.common.model.storage.Record;
package org.opengroup.osdu.dataset.model.request;
import lombok.Data;
@Data
public class GetRecordsResponse {
private List<Record> records;
private List<String> invalidRecords;
private List<String> retryRecords;
public class SchemaExceptionResponse {
private SchemaExceptionResponseBody error;
/** example:
* "{"error":{"code":404,"message":"Schema is not present","errors":[{"domain":"global","reason":"notFound","message":"Schema is not present"}]}}"
*/
}
// Copyright © 2020 Amazon Web Services
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
......@@ -13,20 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//TODO: move this to os-core-common
package org.opengroup.osdu.dataset.model.request;
package org.opengroup.osdu.dataset.storage;
import java.util.List;
import java.util.ArrayList;
import lombok.Data;
@Data
public class CreateUpdateRecordsResponse {
private Integer recordCount;
private List<String> recordIds;
public class SchemaExceptionResponseBody {
private List<String> skippedRecordIds;
}
\ No newline at end of file
private Integer code;
private String message;
private ArrayList<Object> errors;
}
......@@ -17,9 +17,12 @@ package org.opengroup.osdu.dataset.model.response;
import java.util.HashMap;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class DatasetRetrievalDeliveryItem {
String datasetRegistryId;
......@@ -33,5 +36,9 @@ public class DatasetRetrievalDeliveryItem {
this.retrievalProperties = retrievalProperties;
this.providerKey = providerKey;
}
public DatasetRetrievalDeliveryItem() {
//default constructor for serialization/deserialization
}
}
......@@ -28,4 +28,8 @@ public class GetCreateUpdateDatasetRegistryResponse {
public GetCreateUpdateDatasetRegistryResponse(List<Record> datasetRegistries) {
this.datasetRegistries = datasetRegistries;
}
public GetCreateUpdateDatasetRegistryResponse() {
//Default constuctor for serialization/deserialization
}
}
\ No newline at end of file
......@@ -16,9 +16,12 @@ package org.opengroup.osdu.dataset.model.response;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class GetDatasetRetrievalInstructionsResponse {
//Leave as an object. This is different for each DMS
......@@ -26,6 +29,10 @@ public class GetDatasetRetrievalInstructionsResponse {
public GetDatasetRetrievalInstructionsResponse(List<DatasetRetrievalDeliveryItem> delivery) {
this.delivery = delivery;
}
}
public GetDatasetRetrievalInstructionsResponse() {
//default constructor for serialization/deserialization
}
}
......@@ -16,9 +16,12 @@ package org.opengroup.osdu.dataset.model.response;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class GetDatasetStorageInstructionsResponse {
Map<String, Object> storageLocation;
......@@ -28,4 +31,8 @@ public class GetDatasetStorageInstructionsResponse {
this.storageLocation = storageLocation;
this.providerKey = providerKey;
}
public GetDatasetStorageInstructionsResponse() {
//default constructor for serialization/deserialization
}
}
......@@ -22,5 +22,6 @@ public class DmsValidationDoc {
}
public static final String RESOURCE_TYPE_NOT_REGISTERED_ERROR = "No DMS handler for resource type '%s' is registered";
public static final String KIND_SUB_TYPE_NOT_REGISTERED_ERROR = "No DMS handler for kindSubType '%s' is registered";
public static final String DMS_STORAGE_NOT_SUPPORTED_ERROR = "The requested DMS (%s) does not support storage operations";
}
......@@ -14,10 +14,10 @@
//TODO: move to os-core-common
package org.opengroup.osdu.dataset.storage;
package org.opengroup.osdu.dataset.schema;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
public interface IStorageFactory {
IStorageProvider create(DpsHeaders headers);
public interface ISchemaFactory {
ISchemaService create(DpsHeaders headers);
}
\ No newline at end of file
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//TODO: move to os-core-common
package org.opengroup.osdu.dataset.schema;
import org.opengroup.osdu.core.common.model.http.DpsException;
public interface ISchemaService {
Object getSchema(String kind) throws DpsException;
}
\ No newline at end of file
......@@ -15,20 +15,20 @@
//TODO: move to os-core-common
package org.opengroup.osdu.dataset.storage;
package org.opengroup.osdu.dataset.schema;
import lombok.Builder;
import lombok.Data;
@Builder
@Data
public class StorageAPIConfig {
public class SchemaAPIConfig {
@Builder.Default
String rootUrl = "https://os-storage/api/legal/v1";
String rootUrl = "https://os-schema/api/schema-service/v2";
String apiKey;
public static StorageAPIConfig Default() {
return StorageAPIConfig.builder().build();
public static SchemaAPIConfig Default() {
return SchemaAPIConfig.builder().build();
}
}
......@@ -15,28 +15,31 @@
//TODO: move to os-core-common
package org.opengroup.osdu.dataset.storage;
package org.opengroup.osdu.dataset.schema;
import org.opengroup.osdu.core.common.http.HttpClient;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
public class StorageFactory implements IStorageFactory {
public class SchemaFactory implements ISchemaFactory {
private final StorageAPIConfig config;
private final SchemaAPIConfig config;
private final HttpResponseBodyMapper bodyMapper;
public StorageFactory(StorageAPIConfig config) {
public SchemaFactory(SchemaAPIConfig config, HttpResponseBodyMapper bodyMapper) {
if (config == null) {
throw new IllegalArgumentException("StorageAPIConfig cannot be empty");
throw new IllegalArgumentException("SchemaAPIConfig cannot be empty");
}
this.config = config;
this.bodyMapper = bodyMapper;
}
@Override
public IStorageProvider create(DpsHeaders headers) {
public ISchemaService create(DpsHeaders headers) {
if (headers == null) {
throw new NullPointerException("headers cannot be null");
}
return new StorageService(this.config, new HttpClient(), headers);
return new SchemaService(this.config, new HttpClient(), headers, bodyMapper);
}
}
......@@ -15,9 +15,7 @@
//TODO: Move to os-core-common
package org.opengroup.osdu.dataset.storage;
import java.util.List;
package org.opengroup.osdu.dataset.schema;
import com.google.gson.JsonSyntaxException;
......@@ -25,63 +23,48 @@ import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyParsingException;
import org.opengroup.osdu.core.common.model.http.DpsException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.storage.MultiRecordIds;
import org.opengroup.osdu.core.common.model.storage.MultiRecordInfo;
import org.opengroup.osdu.core.common.model.storage.Record;
import org.opengroup.osdu.core.common.model.storage.Schema;
public class StorageService implements IStorageProvider {
public class SchemaService implements ISchemaService {
private final String rootUrl;
private final IHttpClient httpClient;
private final DpsHeaders headers;
private final HttpResponseBodyMapper bodyMapper;
public StorageService(StorageAPIConfig config, IHttpClient httpClient, DpsHeaders headers) {
public SchemaService(SchemaAPIConfig config, IHttpClient httpClient, DpsHeaders headers, HttpResponseBodyMapper bodyMapper) {
this.rootUrl = config.getRootUrl();
this.httpClient = httpClient;
this.headers = headers;
this.bodyMapper = bodyMapper;
if (config.apiKey != null) {
headers.put("AppKey", config.apiKey);
}
}
@Override
public CreateUpdateRecordsResponse createOrUpdateRecords(List<Record> records) throws StorageException {
String url = this.createUrl("/records");
HttpResponse result = this.httpClient
.send(HttpRequest.put(records).url(url).headers(this.headers.getHeaders()).build());
return this.getResult(result, CreateUpdateRecordsResponse.class);
}
@Override
public GetRecordsResponse getRecords(MultiRecordIds ids) throws StorageException {
String url = this.createUrl("/query/records");
HttpResponse result = this.httpClient
.send(HttpRequest.post(ids).url(url).headers(this.headers.getHeaders()).build());
return this.getResult(result, GetRecordsResponse.class);
}
}
@Override
public Schema getSchema(String kind) throws StorageException {
String url = this.createUrl("/schemas/" + kind);
public Object getSchema(String kind) throws DpsException {
String url = this.createUrl("/schema/" + kind);
HttpResponse result = this.httpClient
.send(HttpRequest.get().url(url).headers(this.headers.getHeaders()).build());
return this.getResult(result, Schema.class);
return this.getResult(result, Object.class);
}
private String createUrl(String pathAndQuery) {
return StringUtils.join(this.rootUrl, pathAndQuery);
}
private <T> T getResult(HttpResponse result, Class<T> type) throws StorageException {
private <T> T getResult(HttpResponse result, Class<T> type) throws DpsException {
if (result.isSuccessCode()) {
try {
return result.parseBody(type);
} catch (JsonSyntaxException e) {
throw new StorageException("Error parsing response. Check the inner HttpResponse for more info.",
return bodyMapper.parseBody(result, type);
} catch (HttpResponseBodyParsingException e) {
throw new DpsException("Error parsing response. Check the inner HttpResponse for more info.",
result);
}
} else {
......@@ -89,9 +72,9 @@ public class StorageService implements IStorageProvider {
}
}