Commit 044344eb authored by Matt Wise's avatar Matt Wise
Browse files

Merge branch 'r3-data-definitions' into 'master'

R3 Data Definitions Changes

See merge request !74
parents 9ac42214 ace8d298
Pipeline #25103 passed with stages
in 15 minutes and 19 seconds
......@@ -9,6 +9,20 @@ The following software have components provided under the terms of this license:
- ASM Core (from )
- ASM based accessors helper used by json-smart (from )
- AWS Event Stream (from https://github.com/awslabs/aws-eventstream-java)
- AWS Java SDK :: AWS Core (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Annotations (from )
- AWS Java SDK :: Auth (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Core :: Protocols :: AWS Json Protocol (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Core :: Protocols :: Protocol Core (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: HTTP Client Interface (from )
- AWS Java SDK :: HTTP Clients :: Apache (from )
- AWS Java SDK :: HTTP Clients :: Netty Non-Blocking I/O (from )
- AWS Java SDK :: Profiles (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Regions (from )
- AWS Java SDK :: SDK Core (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Services :: AWS Simple Systems Management (SSM) (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK :: Utilities (from )
- AWS Java SDK for AWS Amplify (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for AWS App Mesh (from https://aws.amazon.com/sdkforjava)
- AWS Java SDK for AWS AppSync (from https://aws.amazon.com/sdkforjava)
......@@ -200,6 +214,7 @@ The following software have components provided under the terms of this license:
- Apache Commons Logging (from http://commons.apache.org/proper/commons-logging/)
- Apache HttpAsyncClient (from http://hc.apache.org/httpcomponents-asyncclient)
- Apache HttpClient (from http://hc.apache.org/httpcomponents-client)
- Apache HttpClient Cache (from http://hc.apache.org/httpcomponents-client)
- Apache HttpCore (from http://hc.apache.org/httpcomponents-core-ga)
- Apache HttpCore NIO (from http://hc.apache.org/httpcomponents-core-ga)
- Apache Log4j API (from )
......@@ -258,17 +273,21 @@ The following software have components provided under the terms of this license:
- Lucene Spatial Extras (from )
- Lucene Suggest (from )
- MapStruct Core (from )
- Mockito (from http://www.mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://mockito.org)
- Mockito (from http://www.mockito.org)
- Mockito (from http://www.mockito.org)
- Netty Reactive Streams HTTP support (from )
- Netty Reactive Streams Implementation (from )
- Netty/Buffer (from http://netty.io/)
- Netty/Codec (from )
- Netty/Codec/HTTP (from )
- Netty/Codec/HTTP2 (from )
- Netty/Common (from )
- Netty/Handler (from )
- Netty/Resolver (from )
- Netty/Transport (from http://netty.io/)
- Netty/Transport/Native/Unix/Common (from )
- Nimbus JOSE+JWT (from https://bitbucket.org/connect2id/nimbus-jose-jwt)
- Nimbus LangTag (from https://bitbucket.org/connect2id/nimbus-language-tags)
- OAuth 2.0 SDK with OpenID Connect extensions (from https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions)
......@@ -308,6 +327,7 @@ The following software have components provided under the terms of this license:
- Spring Web MVC (from https://github.com/spring-projects/spring-framework)
- T-Digest (from https://github.com/tdunning/t-digest)
- aggs-matrix-stats (from https://github.com/elastic/elasticsearch)
- aws-ssm-java-caching-client (from https://github.com/awslabs/aws-ssm-java-caching-client)
- cli (from https://github.com/elastic/elasticsearch)
- compiler (from http://github.com/spullara/mustache.java)
- elasticsearch-core (from https://github.com/elastic/elasticsearch)
......@@ -394,6 +414,13 @@ The following software have components provided under the terms of this license:
- FindBugs-jsr305 (from http://findbugs.sourceforge.net/)
========================================================================
CC0-1.0
========================================================================
The following software have components provided under the terms of this license:
- reactive-streams (from http://www.reactive-streams.org/)
========================================================================
CDDL-1.0
========================================================================
......@@ -581,6 +608,7 @@ public-domain
========================================================================
The following software have components provided under the terms of this license:
- AWS Java SDK :: SDK Core (from https://aws.amazon.com/sdkforjava)
- AWS SDK for Java - Models (from https://aws.amazon.com/sdkforjava)
- Guava: Google Core Libraries for Java (from https://github.com/google/guava.git)
- HdrHistogram (from http://hdrhistogram.github.io/HdrHistogram/)
......@@ -589,6 +617,7 @@ The following software have components provided under the terms of this license:
- Project Lombok (from https://projectlombok.org)
- Project Lombok (from https://projectlombok.org)
- Spring Web (from https://github.com/spring-projects/spring-framework)
- reactive-streams (from http://www.reactive-streams.org/)
========================================================================
unknown
......
......@@ -54,9 +54,9 @@ public class DatasetDmsApi {
@GetMapping("/getStorageInstructions")
@PreAuthorize("@authorizationFilter.hasRole('" + DeliveryRole.VIEWER + "')")
public ResponseEntity<GetDatasetStorageInstructionsResponse> getStorageInstructions(
@RequestParam(value = "resourceType") String resourceType) {
@RequestParam(value = "kindSubType") String kindSubType) {
GetDatasetStorageInstructionsResponse response = this.datasetDmsService.getStorageInstructions(resourceType);
GetDatasetStorageInstructionsResponse response = this.datasetDmsService.getStorageInstructions(kindSubType);
return new ResponseEntity<GetDatasetStorageInstructionsResponse>(response, HttpStatus.OK);
}
......
......@@ -15,9 +15,12 @@
package org.opengroup.osdu.dataset.di;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
import org.opengroup.osdu.core.common.entitlements.EntitlementsFactory;
import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
......@@ -25,6 +28,9 @@ import org.springframework.stereotype.Component;
@Component
public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlementsFactory> {
private final ObjectMapper objectMapper = new ObjectMapper();
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
@Value("${AUTHORIZE_API}")
private String AUTHORIZE_API;
......@@ -38,7 +44,8 @@ public class EntitlementsClientFactory extends AbstractFactoryBean<IEntitlements
.builder()
.rootUrl(AUTHORIZE_API)
.apiKey(AUTHORIZE_API_KEY)
.build());
.build(),
bodyMapper);
}
@Override
......
......@@ -13,22 +13,38 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//TODO: Move to os-core-common
package org.opengroup.osdu.dataset.di;
package org.opengroup.osdu.dataset.storage;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.Data;
import lombok.EqualsAndHashCode;
import org.opengroup.osdu.core.common.model.http.DpsException;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.dataset.schema.ISchemaFactory;
import org.opengroup.osdu.dataset.schema.SchemaAPIConfig;
import org.opengroup.osdu.dataset.schema.SchemaFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
@Data
@EqualsAndHashCode(callSuper = false)
public class StorageException extends DpsException {
@Component
public class SchemaClientFactory extends AbstractFactoryBean<ISchemaFactory> {
private static final long serialVersionUID = 9094949225576291097L;
private final ObjectMapper objectMapper = new ObjectMapper();
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
public StorageException(String message, HttpResponse httpResponse) {
super(message, httpResponse);
}
@Value("${SCHEMA_API}")
private String SCHEMA_API;
@Override
public Class<?> getObjectType() {
return ISchemaFactory.class;
}
@Override
protected ISchemaFactory createInstance() throws Exception {
return new SchemaFactory(SchemaAPIConfig
.builder()
.rootUrl(SCHEMA_API)
.build(),
bodyMapper);
}
}
\ No newline at end of file
......@@ -15,12 +15,12 @@
package org.opengroup.osdu.dataset.di;
import org.opengroup.osdu.core.common.legal.ILegalFactory;
import org.opengroup.osdu.core.common.legal.LegalAPIConfig;
import org.opengroup.osdu.core.common.legal.LegalFactory;
import org.opengroup.osdu.dataset.storage.IStorageFactory;
import org.opengroup.osdu.dataset.storage.StorageAPIConfig;
import org.opengroup.osdu.dataset.storage.StorageFactory;
import com.fasterxml.jackson.databind.ObjectMapper;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.storage.IStorageFactory;
import org.opengroup.osdu.core.common.storage.StorageAPIConfig;
import org.opengroup.osdu.core.common.storage.StorageFactory;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.beans.factory.config.AbstractFactoryBean;
import org.springframework.stereotype.Component;
......@@ -28,6 +28,9 @@ import org.springframework.stereotype.Component;
@Component
public class StorageClientFactory extends AbstractFactoryBean<IStorageFactory> {
private final ObjectMapper objectMapper = new ObjectMapper();
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
@Value("${STORAGE_API}")
private String STORAGE_API;
......@@ -40,7 +43,8 @@ public class StorageClientFactory extends AbstractFactoryBean<IStorageFactory> {
protected IStorageFactory createInstance() throws Exception {
return new StorageFactory(StorageAPIConfig
.builder()
.rootUrl(STORAGE_API)
.build());
.rootUrl(STORAGE_API)
.build(),
bodyMapper);
}
}
\ No newline at end of file
......@@ -17,12 +17,16 @@
package org.opengroup.osdu.dataset.dms;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.gson.JsonSyntaxException;
import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyParsingException;
import org.opengroup.osdu.core.common.model.http.AppException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.dataset.model.request.GetDatasetRegistryRequest;
......@@ -31,17 +35,19 @@ import org.opengroup.osdu.dataset.model.response.GetDatasetStorageInstructionsRe
import org.springframework.http.HttpStatus;
public class DmsService implements IDmsProvider {
private String dmsServiceUrl;
private DmsServiceProperties dmsServiceProperties;
private final IHttpClient httpClient;
private final DpsHeaders headers;
private final ObjectMapper objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false);
private final HttpResponseBodyMapper bodyMapper = new HttpResponseBodyMapper(objectMapper);
public DmsService(DmsServiceProperties dmsServiceProperties, IHttpClient httpClient, DpsHeaders headers) {
this.dmsServiceProperties = dmsServiceProperties;
this.dmsServiceUrl = dmsServiceProperties.getDmsServiceBaseUrl();
this.dmsServiceUrl = dmsServiceProperties.getDmsServiceBaseUrl();
this.httpClient = httpClient;
this.headers = headers;
......@@ -61,7 +67,8 @@ public class DmsService implements IDmsProvider {
}
@Override
public GetDatasetRetrievalInstructionsResponse getDatasetRetrievalInstructions(GetDatasetRegistryRequest datasetRegistryRequest) throws DmsException {
public GetDatasetRetrievalInstructionsResponse getDatasetRetrievalInstructions(
GetDatasetRegistryRequest datasetRegistryRequest) throws DmsException {
String url = this.createUrl("/getRetrievalInstructions");
HttpResponse result = this.httpClient
......@@ -69,7 +76,6 @@ public class DmsService implements IDmsProvider {
return this.getResult(result, GetDatasetRetrievalInstructionsResponse.class);
}
private String createUrl(String requestPathAndQuery) {
return StringUtils.join(this.dmsServiceUrl, requestPathAndQuery);
}
......@@ -77,8 +83,8 @@ public class DmsService implements IDmsProvider {
private <T> T getResult(HttpResponse result, Class<T> type) throws DmsException {
if (result.isSuccessCode()) {
try {
return result.parseBody(type);
} catch (JsonSyntaxException e) {
return bodyMapper.parseBody(result, type);
} catch (JsonSyntaxException | HttpResponseBodyParsingException e) {
throw new DmsException("Error parsing response. Check the inner HttpResponse for more info.",
result);
}
......
// Copyright © 2020 Amazon Web Services
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
......@@ -13,19 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//TODO: move this to os-core-common
package org.opengroup.osdu.dataset.storage;
import java.util.List;
import org.opengroup.osdu.core.common.model.storage.Record;
package org.opengroup.osdu.dataset.model.request;
import lombok.Data;
@Data
public class GetRecordsResponse {
private List<Record> records;
private List<String> invalidRecords;
private List<String> retryRecords;
public class SchemaExceptionResponse {
private SchemaExceptionResponseBody error;
/** example:
* "{"error":{"code":404,"message":"Schema is not present","errors":[{"domain":"global","reason":"notFound","message":"Schema is not present"}]}}"
*/
}
// Copyright © 2020 Amazon Web Services
// Copyright 2017-2019, Schlumberger
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
......@@ -13,20 +12,17 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//TODO: move this to os-core-common
package org.opengroup.osdu.dataset.model.request;
package org.opengroup.osdu.dataset.storage;
import java.util.List;
import java.util.ArrayList;
import lombok.Data;
@Data
public class CreateUpdateRecordsResponse {
private Integer recordCount;
private List<String> recordIds;
public class SchemaExceptionResponseBody {
private List<String> skippedRecordIds;
}
\ No newline at end of file
private Integer code;
private String message;
private ArrayList<Object> errors;
}
......@@ -17,9 +17,12 @@ package org.opengroup.osdu.dataset.model.response;
import java.util.HashMap;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class DatasetRetrievalDeliveryItem {
String datasetRegistryId;
......@@ -33,5 +36,9 @@ public class DatasetRetrievalDeliveryItem {
this.retrievalProperties = retrievalProperties;
this.providerKey = providerKey;
}
public DatasetRetrievalDeliveryItem() {
//default constructor for serialization/deserialization
}
}
......@@ -28,4 +28,8 @@ public class GetCreateUpdateDatasetRegistryResponse {
public GetCreateUpdateDatasetRegistryResponse(List<Record> datasetRegistries) {
this.datasetRegistries = datasetRegistries;
}
public GetCreateUpdateDatasetRegistryResponse() {
//Default constuctor for serialization/deserialization
}
}
\ No newline at end of file
......@@ -16,9 +16,12 @@ package org.opengroup.osdu.dataset.model.response;
import java.util.List;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class GetDatasetRetrievalInstructionsResponse {
//Leave as an object. This is different for each DMS
......@@ -26,6 +29,10 @@ public class GetDatasetRetrievalInstructionsResponse {
public GetDatasetRetrievalInstructionsResponse(List<DatasetRetrievalDeliveryItem> delivery) {
this.delivery = delivery;
}
}
public GetDatasetRetrievalInstructionsResponse() {
//default constructor for serialization/deserialization
}
}
......@@ -16,9 +16,12 @@ package org.opengroup.osdu.dataset.model.response;
import java.util.Map;
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import lombok.Data;
@Data
@JsonIgnoreProperties(ignoreUnknown = true)
public class GetDatasetStorageInstructionsResponse {
Map<String, Object> storageLocation;
......@@ -28,4 +31,8 @@ public class GetDatasetStorageInstructionsResponse {
this.storageLocation = storageLocation;
this.providerKey = providerKey;
}
public GetDatasetStorageInstructionsResponse() {
//default constructor for serialization/deserialization
}
}
......@@ -22,5 +22,6 @@ public class DmsValidationDoc {
}
public static final String RESOURCE_TYPE_NOT_REGISTERED_ERROR = "No DMS handler for resource type '%s' is registered";
public static final String KIND_SUB_TYPE_NOT_REGISTERED_ERROR = "No DMS handler for kindSubType '%s' is registered";
public static final String DMS_STORAGE_NOT_SUPPORTED_ERROR = "The requested DMS (%s) does not support storage operations";
}
......@@ -14,10 +14,10 @@
//TODO: move to os-core-common
package org.opengroup.osdu.dataset.storage;
package org.opengroup.osdu.dataset.schema;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
public interface IStorageFactory {
IStorageProvider create(DpsHeaders headers);
public interface ISchemaFactory {
ISchemaService create(DpsHeaders headers);
}
\ No newline at end of file
// Copyright © 2020 Amazon Web Services
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//TODO: move to os-core-common
package org.opengroup.osdu.dataset.schema;
import org.opengroup.osdu.core.common.model.http.DpsException;
public interface ISchemaService {
Object getSchema(String kind) throws DpsException;
}
\ No newline at end of file
......@@ -15,20 +15,20 @@
//TODO: move to os-core-common
package org.opengroup.osdu.dataset.storage;
package org.opengroup.osdu.dataset.schema;
import lombok.Builder;
import lombok.Data;
@Builder
@Data
public class StorageAPIConfig {
public class SchemaAPIConfig {
@Builder.Default
String rootUrl = "https://os-storage/api/legal/v1";
String rootUrl = "https://os-schema/api/schema-service/v2";
String apiKey;
public static StorageAPIConfig Default() {
return StorageAPIConfig.builder().build();
public static SchemaAPIConfig Default() {
return SchemaAPIConfig.builder().build();
}
}
......@@ -15,28 +15,31 @@
//TODO: move to os-core-common
package org.opengroup.osdu.dataset.storage;
package org.opengroup.osdu.dataset.schema;
import org.opengroup.osdu.core.common.http.HttpClient;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
public class StorageFactory implements IStorageFactory {
public class SchemaFactory implements ISchemaFactory {
private final StorageAPIConfig config;
private final SchemaAPIConfig config;
private final HttpResponseBodyMapper bodyMapper;
public StorageFactory(StorageAPIConfig config) {
public SchemaFactory(SchemaAPIConfig config, HttpResponseBodyMapper bodyMapper) {
if (config == null) {
throw new IllegalArgumentException("StorageAPIConfig cannot be empty");
throw new IllegalArgumentException("SchemaAPIConfig cannot be empty");
}
this.config = config;
this.bodyMapper = bodyMapper;
}
@Override
public IStorageProvider create(DpsHeaders headers) {
public ISchemaService create(DpsHeaders headers) {
if (headers == null) {
throw new NullPointerException("headers cannot be null");
}
return new StorageService(this.config, new HttpClient(), headers);
return new SchemaService(this.config, new HttpClient(), headers, bodyMapper);
}
}
......@@ -15,9 +15,7 @@
//TODO: Move to os-core-common
package org.opengroup.osdu.dataset.storage;
import java.util.List;
package org.opengroup.osdu.dataset.schema;
import com.google.gson.JsonSyntaxException;
......@@ -25,63 +23,48 @@ import org.apache.commons.lang3.StringUtils;
import org.opengroup.osdu.core.common.http.HttpRequest;
import org.opengroup.osdu.core.common.http.HttpResponse;
import org.opengroup.osdu.core.common.http.IHttpClient;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
import org.opengroup.osdu.core.common.http.json.HttpResponseBodyParsingException;
import org.opengroup.osdu.core.common.model.http.DpsException;
import org.opengroup.osdu.core.common.model.http.DpsHeaders;
import org.opengroup.osdu.core.common.model.storage.MultiRecordIds;
import org.opengroup.osdu.core.common.model.storage.MultiRecordInfo;