Azure changes for file collection feature
Integrating with DataLakeStore exists in core lib Azure. implement stroageInstruction and retrivalInstruction.
Merge request reports
Activity
requested review from @krveduru, @kibattul, and @harshit283
assigned to @hsaxena
mentioned in merge request !218 (merged)
15 */ 16 17 package org.opengroup.osdu.file.provider.azure.config; 18 19 import com.azure.storage.file.datalake.DataLakeServiceClient; 20 import org.opengroup.osdu.azure.datalakestorage.IDataLakeClientFactory; 21 import org.opengroup.osdu.common.Validators; 22 import org.opengroup.osdu.core.common.model.http.DpsHeaders; 23 import org.springframework.beans.factory.annotation.Autowired; 24 import org.springframework.stereotype.Component; 25 import org.springframework.web.context.annotation.RequestScope; 26 27 import javax.inject.Inject; 28 29 @Component 30 @RequestScope 23 import org.springframework.beans.factory.annotation.Autowired; 24 import org.springframework.stereotype.Component; 25 import org.springframework.web.context.annotation.RequestScope; 26 27 import javax.inject.Inject; 28 29 @Component 30 @RequestScope 31 public class DataLakeClientWrapper { 32 33 private final String storageAccount; 34 35 @Autowired 36 private IDataLakeClientFactory dataLakeClientFactory; 37 38 @Inject changed this line in version 4 of the diff
13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package org.opengroup.osdu.file.provider.azure.model; 18 19 import lombok.AllArgsConstructor; 20 import lombok.Builder; 21 import lombok.Data; 22 import lombok.NoArgsConstructor; 23 24 @Data 25 @Builder 26 @NoArgsConstructor 27 @AllArgsConstructor 28 public class AzureFileCollectionDmsUploadLocation { @kibattul During File Service implementation it was requested that although both the responses look similar currently, to be able to extend them as needed in future, we decided to have two different models for Upload and Download respones.
34 public class DataLakeRepository implements IStorageRepository { 35 36 @Autowired 37 DataLakeStore dataLakeStore; 38 39 @Autowired 40 DpsHeaders dpsHeaders; 41 42 /** 43 * Creates the empty object blob in bucket by filepath. 44 * 45 * @param directoryName file path 46 * @return info created blob and signed URL 47 */ 48 @Override 49 @SneakyThrows 35 36 @Autowired 37 DataLakeStore dataLakeStore; 38 39 @Autowired 40 DpsHeaders dpsHeaders; 41 42 /** 43 * Creates the empty object blob in bucket by filepath. 44 * 45 * @param directoryName file path 46 * @return info created blob and signed URL 47 */ 48 @Override 49 @SneakyThrows 50 public SignedObject createSignedObject(String fileSystemName, String directoryName) { changed this line in version 4 of the diff
93 DataLakeConfig dataLakeConfig; 94 95 /** 96 * Creates the empty object blob in storage. 97 * Bucket name is determined by tenant using {@code partitionID}. 98 * Object name is concat of a filepath and a fileID. Filepath is determined by user. 99 * 100 * @param directoryId directory ID 101 * @param authorizationToken authorization token 102 * @param partitionID partition ID 103 * @return info about object URI, signed URL and when and who created blob. 104 */ 105 @Override 106 public SignedUrl createSignedUrl(String directoryId, String authorizationToken, String partitionID) { 107 log.debug("Creating the signed url for directoryId : {}. Authorization : {}, partitionID : {}", 108 directoryId, authorizationToken, partitionID); changed this line in version 4 of the diff
131 } 132 133 private String getRelativeFileSource(String directoryName) { 134 return "/" + directoryName; 135 } 136 137 /** 138 * Generates Signed URL for File Upload Operations in DMS API Context. 139 * 140 * @param directoryID directoryID 141 * @param partitionID partition ID 142 * @return info about object URI, upload signed URL etc. 143 */ 144 @Override 145 public StorageInstructionsResponse createStorageInstructions(String directoryID, String partitionID) { 146 SignedUrl signedUrl = this.createSignedUrl(directoryID, dpsHeaders.getAuthorization(), partitionID); changed this line in version 10 of the diff
25 25 @Component 26 26 public class ServiceHelper { 27 27 28 private static final String CONTAINER_NAME = "containerName"; 29 private static final String FILE_SYSTEM_NAME = "fileSystemName"; 30 private static final String FILE_PATH = "filePath"; 31 private static final String DIRECTORY_PATH = "directoryPath"; 32 28 33 // refer https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftstorage for naming convention 29 34 private final String absoluteFilePathPattern = "https://[a-z0-9][a-z0-9^-]*.blob.core.windows.net/(?<containerName>[^/]*)/(?<filePath>.*)"; 35 private final String absoluteDirectoryPathPattern = "https://[a-z0-9][a-z0-9^-]*..dfs.core.windows.net/(?<fileSystemName>[^/]*)/(?<directoryPath>.*)"; changed this line in version 4 of the diff
25 25 import org.opengroup.osdu.file.provider.azure.config.BlobServiceClientWrapper; 26 26 import org.opengroup.osdu.file.provider.interfaces.IStorageUtilService; 27 27 import org.springframework.beans.factory.annotation.Autowired; 28 import org.springframework.context.annotation.Primary; 28 29 import org.springframework.stereotype.Service; 29 30 30 31 31 @Service 32 32 @Slf4j 33 33 @RequiredArgsConstructor 34 @Primary It is happening because we have two implementations of the same interface IStorageUtilService. If we use named based beans...we need to change FileMetadataService class in file core and other providers code will also be affected.
Edited by Harshit Saxena
8 * https://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package org.opengroup.osdu.file.provider.azure.util; 18 19 import org.apache.commons.lang3.StringUtils; 20 import org.springframework.stereotype.Component; 21 22 @Component 23 public class FilePathUtil { @hsaxena Can you resolve merge conflicts on this MR, also please rebase it
Done @harshit283 . Please have a look.