From 741a69df9d6a24126410c554b373967c524dac8d Mon Sep 17 00:00:00 2001 From: Krishna Nikhil Vedurumudi Date: Mon, 5 Jul 2021 22:09:09 +0530 Subject: [PATCH 1/3] Added common interfaces and models for DMS APIs --- .../osdu/core/common/dms/IDmsService.java | 37 +++++++++++++++++++ .../dms/model/DatasetRetrievalProperties.java | 33 +++++++++++++++++ .../model/RetrievalInstructionsRequest.java | 26 +++++++++++++ .../model/RetrievalInstructionsResponse.java | 33 +++++++++++++++++ .../model/StorageInstructionsResponse.java | 33 +++++++++++++++++ 5 files changed, 162 insertions(+) create mode 100644 src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java create mode 100644 src/main/java/org/opengroup/osdu/core/common/dms/model/DatasetRetrievalProperties.java create mode 100644 src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsRequest.java create mode 100644 src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsResponse.java create mode 100644 src/main/java/org/opengroup/osdu/core/common/dms/model/StorageInstructionsResponse.java diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java b/src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java new file mode 100644 index 0000000..0de0aaa --- /dev/null +++ b/src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java @@ -0,0 +1,37 @@ +/* + * Copyright 2021 Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.core.common.dms; + +import org.opengroup.osdu.core.common.dms.model.RetrievalInstructionsRequest; +import org.opengroup.osdu.core.common.dms.model.RetrievalInstructionsResponse; +import org.opengroup.osdu.core.common.dms.model.StorageInstructionsResponse; + +public interface IDmsService { + /** + * Method is used to generate storage instructions for datasets. Each DMS or CSP could have different implementations but should adhere to this interface. + * The storage instructions response include the Signed URL / Temporary credentials along with other CSP specific metadata required to identify a Dataset. + * @return StorageInstructionsResponse + */ + StorageInstructionsResponse getStorageInstructions(); + + /** + * Method is used to generate retrieval instructions for datasets. Each DMS or CSP could have different implementations but should adhere to this interface. + * The storage instructions response include the Signed URL / Temporary credentials along with other CSP specific metadata required to identify a Dataset. + * @return StorageInstructionsResponse + */ + RetrievalInstructionsResponse getRetrievalInstructions(RetrievalInstructionsRequest retrievalInstructionsRequest); +} diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/model/DatasetRetrievalProperties.java b/src/main/java/org/opengroup/osdu/core/common/dms/model/DatasetRetrievalProperties.java new file mode 100644 index 0000000..dd55c26 --- /dev/null +++ b/src/main/java/org/opengroup/osdu/core/common/dms/model/DatasetRetrievalProperties.java @@ -0,0 +1,33 @@ +/* + * Copyright 2021 Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.core.common.dms.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Map; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class DatasetRetrievalProperties { + private String datasetRegistryId; + private Map retrievalProperties; +} diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsRequest.java b/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsRequest.java new file mode 100644 index 0000000..a9c81ca --- /dev/null +++ b/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsRequest.java @@ -0,0 +1,26 @@ +/* + * Copyright 2021 Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.core.common.dms.model; + +import lombok.Data; + +import java.util.List; + +@Data +public class RetrievalInstructionsRequest { + public List datasetRegistryIds; +} diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsResponse.java b/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsResponse.java new file mode 100644 index 0000000..f6556b4 --- /dev/null +++ b/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsResponse.java @@ -0,0 +1,33 @@ +/* + * Copyright 2021 Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.core.common.dms.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class RetrievalInstructionsResponse { + private String providerKey; + private List datasets; +} diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/model/StorageInstructionsResponse.java b/src/main/java/org/opengroup/osdu/core/common/dms/model/StorageInstructionsResponse.java new file mode 100644 index 0000000..44590be --- /dev/null +++ b/src/main/java/org/opengroup/osdu/core/common/dms/model/StorageInstructionsResponse.java @@ -0,0 +1,33 @@ +/* + * Copyright 2021 Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.core.common.dms.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Map; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class StorageInstructionsResponse { + private String providerKey; + private Map storageLocation; +} -- GitLab From 30eda6da954c214a0387ffaae61b8cdbb7125a5f Mon Sep 17 00:00:00 2001 From: Krishna Nikhil Vedurumudi Date: Mon, 12 Jul 2021 13:28:10 +0530 Subject: [PATCH 2/3] Added Copy DMS API to the IDmsService --- .../osdu/core/common/dms/IDmsService.java | 17 ++++++--- .../dms/constants/DatasetConstants.java | 27 ++++++++++++++ .../core/common/dms/model/CopyDmsRequest.java | 36 +++++++++++++++++++ .../common/dms/model/CopyDmsResponse.java | 32 +++++++++++++++++ .../model/RetrievalInstructionsRequest.java | 7 ++++ .../model/RetrievalInstructionsResponse.java | 7 ++-- 6 files changed, 120 insertions(+), 6 deletions(-) create mode 100644 src/main/java/org/opengroup/osdu/core/common/dms/constants/DatasetConstants.java create mode 100644 src/main/java/org/opengroup/osdu/core/common/dms/model/CopyDmsRequest.java create mode 100644 src/main/java/org/opengroup/osdu/core/common/dms/model/CopyDmsResponse.java diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java b/src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java index 0de0aaa..efc7d0c 100644 --- a/src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java +++ b/src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java @@ -16,9 +16,10 @@ package org.opengroup.osdu.core.common.dms; -import org.opengroup.osdu.core.common.dms.model.RetrievalInstructionsRequest; -import org.opengroup.osdu.core.common.dms.model.RetrievalInstructionsResponse; -import org.opengroup.osdu.core.common.dms.model.StorageInstructionsResponse; +import org.opengroup.osdu.core.common.dms.model.*; +import org.opengroup.osdu.core.common.model.storage.Record; + +import java.util.List; public interface IDmsService { /** @@ -31,7 +32,15 @@ public interface IDmsService { /** * Method is used to generate retrieval instructions for datasets. Each DMS or CSP could have different implementations but should adhere to this interface. * The storage instructions response include the Signed URL / Temporary credentials along with other CSP specific metadata required to identify a Dataset. - * @return StorageInstructionsResponse + * @param retrievalInstructionsRequest Request containing dataset ids which should be downloaded + * @return RetrievalInstructionsResponse */ RetrievalInstructionsResponse getRetrievalInstructions(RetrievalInstructionsRequest retrievalInstructionsRequest); + + /** + * Method is used to Copy Datasets from staging locations to persistent locations + * @param datasetSources Request containing list of Dataset paths that needs to copied to persistent location + * @return CopyDmsResponse + */ + List copyDatasetsToPersistentLocation(List datasetSources); } diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/constants/DatasetConstants.java b/src/main/java/org/opengroup/osdu/core/common/dms/constants/DatasetConstants.java new file mode 100644 index 0000000..6ffdf27 --- /dev/null +++ b/src/main/java/org/opengroup/osdu/core/common/dms/constants/DatasetConstants.java @@ -0,0 +1,27 @@ +/* + * Copyright 2021 Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.core.common.dms.constants; + +public final class DatasetConstants { + private DatasetConstants() { + // Avoid initialization + } + + // Roles required to perform read and write operations on Datasets. + public static final String DATASET_VIEWER_ROLE = "service.dataset.viewers"; + public static final String DATASET_EDITOR_ROLE = "service.dataset.editors"; +} diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/model/CopyDmsRequest.java b/src/main/java/org/opengroup/osdu/core/common/dms/model/CopyDmsRequest.java new file mode 100644 index 0000000..166150c --- /dev/null +++ b/src/main/java/org/opengroup/osdu/core/common/dms/model/CopyDmsRequest.java @@ -0,0 +1,36 @@ +/* + * Copyright 2021 Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.core.common.dms.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import org.opengroup.osdu.core.common.model.storage.Record; + +import java.util.ArrayList; +import java.util.List; + +@Data +@Builder +@AllArgsConstructor +public class CopyDmsRequest { + List datasetSources; + + public CopyDmsRequest() { + this.datasetSources = new ArrayList<>(); + } +} diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/model/CopyDmsResponse.java b/src/main/java/org/opengroup/osdu/core/common/dms/model/CopyDmsResponse.java new file mode 100644 index 0000000..2b0d218 --- /dev/null +++ b/src/main/java/org/opengroup/osdu/core/common/dms/model/CopyDmsResponse.java @@ -0,0 +1,32 @@ +/* + * Copyright 2021 Microsoft Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.opengroup.osdu.core.common.dms.model; + +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.opengroup.osdu.core.common.model.storage.Record; + +@Data +@Builder +@NoArgsConstructor +@AllArgsConstructor +public class CopyDmsResponse { + private boolean success; + private String datasetBlobStoragePath; +} diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsRequest.java b/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsRequest.java index a9c81ca..5f92eeb 100644 --- a/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsRequest.java +++ b/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsRequest.java @@ -16,11 +16,18 @@ package org.opengroup.osdu.core.common.dms.model; +import lombok.AllArgsConstructor; import lombok.Data; +import java.util.ArrayList; import java.util.List; @Data +@AllArgsConstructor public class RetrievalInstructionsRequest { public List datasetRegistryIds; + + public RetrievalInstructionsRequest() { + this.datasetRegistryIds = new ArrayList<>(); + } } diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsResponse.java b/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsResponse.java index f6556b4..e567900 100644 --- a/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsResponse.java +++ b/src/main/java/org/opengroup/osdu/core/common/dms/model/RetrievalInstructionsResponse.java @@ -19,15 +19,18 @@ package org.opengroup.osdu.core.common.dms.model; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; -import lombok.NoArgsConstructor; +import java.util.ArrayList; import java.util.List; @Data @Builder -@NoArgsConstructor @AllArgsConstructor public class RetrievalInstructionsResponse { private String providerKey; private List datasets; + + public RetrievalInstructionsResponse() { + this.datasets = new ArrayList<>(); + } } -- GitLab From 2314723d0154841ed533900f9bad8db699d23dbb Mon Sep 17 00:00:00 2001 From: Krishna Nikhil Vedurumudi Date: Tue, 13 Jul 2021 16:20:51 +0530 Subject: [PATCH 3/3] Updated Javadoc as per review comments --- .../opengroup/osdu/core/common/dms/IDmsService.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java b/src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java index efc7d0c..c116cbe 100644 --- a/src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java +++ b/src/main/java/org/opengroup/osdu/core/common/dms/IDmsService.java @@ -21,16 +21,20 @@ import org.opengroup.osdu.core.common.model.storage.Record; import java.util.List; +/** + * Interface for different DMS functionalities. + * Each DMS or CSP could have different implementations but should adhere to this interface. + */ public interface IDmsService { /** - * Method is used to generate storage instructions for datasets. Each DMS or CSP could have different implementations but should adhere to this interface. + * Method is used to generate storage instructions for datasets. * The storage instructions response include the Signed URL / Temporary credentials along with other CSP specific metadata required to identify a Dataset. * @return StorageInstructionsResponse */ StorageInstructionsResponse getStorageInstructions(); /** - * Method is used to generate retrieval instructions for datasets. Each DMS or CSP could have different implementations but should adhere to this interface. + * Method is used to generate retrieval instructions for datasets. * The storage instructions response include the Signed URL / Temporary credentials along with other CSP specific metadata required to identify a Dataset. * @param retrievalInstructionsRequest Request containing dataset ids which should be downloaded * @return RetrievalInstructionsResponse @@ -39,8 +43,9 @@ public interface IDmsService { /** * Method is used to Copy Datasets from staging locations to persistent locations - * @param datasetSources Request containing list of Dataset paths that needs to copied to persistent location - * @return CopyDmsResponse + * @param datasetSources Request containing list of Dataset Metadata Records from which DMS implementations + * will parse Dataset path in Cloud Blob Stores and copies content to persistent locations. + * @return Copy operation responses that contain if operation is successful and the location to which the dataset is copied to. */ List copyDatasetsToPersistentLocation(List datasetSources); } -- GitLab