Skip to content
Snippets Groups Projects

Azure changes for file collection feature

Merged Harshit Saxena requested to merge file_collection_azure into master

Integrating with DataLakeStore exists in core lib Azure. implement stroageInstruction and retrivalInstruction.

Merge request reports

Merge request pipeline #95707 failed

Merge request pipeline failed for b7e39233

Merged by Harshit SaxenaHarshit Saxena 2 years ago (Feb 24, 2022 5:18pm UTC)

Loading

Pipeline #95951 passed

Pipeline passed for 7caadd78 on master

Test coverage 27.00% from 0 jobs
Deployed to G‎CP‎ 2 years ago
Deployed to I‎BM‎ 2 years ago
Deployed to A‎WS‎ 2 years ago

Activity

Filter activity
  • Approvals
  • Assignees & reviewers
  • Comments (from bots)
  • Comments (from users)
  • Commits & branches
  • Edits
  • Labels
  • Lock status
  • Mentions
  • Merge request status
  • Tracking
15 */
16
17 package org.opengroup.osdu.file.provider.azure.config;
18
19 import com.azure.storage.file.datalake.DataLakeServiceClient;
20 import org.opengroup.osdu.azure.datalakestorage.IDataLakeClientFactory;
21 import org.opengroup.osdu.common.Validators;
22 import org.opengroup.osdu.core.common.model.http.DpsHeaders;
23 import org.springframework.beans.factory.annotation.Autowired;
24 import org.springframework.stereotype.Component;
25 import org.springframework.web.context.annotation.RequestScope;
26
27 import javax.inject.Inject;
28
29 @Component
30 @RequestScope
  • 23 import org.springframework.beans.factory.annotation.Autowired;
    24 import org.springframework.stereotype.Component;
    25 import org.springframework.web.context.annotation.RequestScope;
    26
    27 import javax.inject.Inject;
    28
    29 @Component
    30 @RequestScope
    31 public class DataLakeClientWrapper {
    32
    33 private final String storageAccount;
    34
    35 @Autowired
    36 private IDataLakeClientFactory dataLakeClientFactory;
    37
    38 @Inject
  • 13 * See the License for the specific language governing permissions and
    14 * limitations under the License.
    15 */
    16
    17 package org.opengroup.osdu.file.provider.azure.model;
    18
    19 import lombok.AllArgsConstructor;
    20 import lombok.Builder;
    21 import lombok.Data;
    22 import lombok.NoArgsConstructor;
    23
    24 @Data
    25 @Builder
    26 @NoArgsConstructor
    27 @AllArgsConstructor
    28 public class AzureFileCollectionDmsUploadLocation {
  • 34 public class DataLakeRepository implements IStorageRepository {
    35
    36 @Autowired
    37 DataLakeStore dataLakeStore;
    38
    39 @Autowired
    40 DpsHeaders dpsHeaders;
    41
    42 /**
    43 * Creates the empty object blob in bucket by filepath.
    44 *
    45 * @param directoryName file path
    46 * @return info created blob and signed URL
    47 */
    48 @Override
    49 @SneakyThrows
  • 35
    36 @Autowired
    37 DataLakeStore dataLakeStore;
    38
    39 @Autowired
    40 DpsHeaders dpsHeaders;
    41
    42 /**
    43 * Creates the empty object blob in bucket by filepath.
    44 *
    45 * @param directoryName file path
    46 * @return info created blob and signed URL
    47 */
    48 @Override
    49 @SneakyThrows
    50 public SignedObject createSignedObject(String fileSystemName, String directoryName) {
  • 93 DataLakeConfig dataLakeConfig;
    94
    95 /**
    96 * Creates the empty object blob in storage.
    97 * Bucket name is determined by tenant using {@code partitionID}.
    98 * Object name is concat of a filepath and a fileID. Filepath is determined by user.
    99 *
    100 * @param directoryId directory ID
    101 * @param authorizationToken authorization token
    102 * @param partitionID partition ID
    103 * @return info about object URI, signed URL and when and who created blob.
    104 */
    105 @Override
    106 public SignedUrl createSignedUrl(String directoryId, String authorizationToken, String partitionID) {
    107 log.debug("Creating the signed url for directoryId : {}. Authorization : {}, partitionID : {}",
    108 directoryId, authorizationToken, partitionID);
  • 131 }
    132
    133 private String getRelativeFileSource(String directoryName) {
    134 return "/" + directoryName;
    135 }
    136
    137 /**
    138 * Generates Signed URL for File Upload Operations in DMS API Context.
    139 *
    140 * @param directoryID directoryID
    141 * @param partitionID partition ID
    142 * @return info about object URI, upload signed URL etc.
    143 */
    144 @Override
    145 public StorageInstructionsResponse createStorageInstructions(String directoryID, String partitionID) {
    146 SignedUrl signedUrl = this.createSignedUrl(directoryID, dpsHeaders.getAuthorization(), partitionID);
  • 25 25 @Component
    26 26 public class ServiceHelper {
    27 27
    28 private static final String CONTAINER_NAME = "containerName";
    29 private static final String FILE_SYSTEM_NAME = "fileSystemName";
    30 private static final String FILE_PATH = "filePath";
    31 private static final String DIRECTORY_PATH = "directoryPath";
    32
    28 33 // refer https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftstorage for naming convention
    29 34 private final String absoluteFilePathPattern = "https://[a-z0-9][a-z0-9^-]*.blob.core.windows.net/(?<containerName>[^/]*)/(?<filePath>.*)";
    35 private final String absoluteDirectoryPathPattern = "https://[a-z0-9][a-z0-9^-]*..dfs.core.windows.net/(?<fileSystemName>[^/]*)/(?<directoryPath>.*)";
  • 60 61 @Service
    61 62 @Slf4j
    62 63 @AllArgsConstructor
    64 @Primary
  • 25 25 import org.opengroup.osdu.file.provider.azure.config.BlobServiceClientWrapper;
    26 26 import org.opengroup.osdu.file.provider.interfaces.IStorageUtilService;
    27 27 import org.springframework.beans.factory.annotation.Autowired;
    28 import org.springframework.context.annotation.Primary;
    28 29 import org.springframework.stereotype.Service;
    29 30
    30
    31 31 @Service
    32 32 @Slf4j
    33 33 @RequiredArgsConstructor
    34 @Primary
  • 8 * https://www.apache.org/licenses/LICENSE-2.0
    9 *
    10 * Unless required by applicable law or agreed to in writing, software
    11 * distributed under the License is distributed on an "AS IS" BASIS,
    12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13 * See the License for the specific language governing permissions and
    14 * limitations under the License.
    15 */
    16
    17 package org.opengroup.osdu.file.provider.azure.util;
    18
    19 import org.apache.commons.lang3.StringUtils;
    20 import org.springframework.stereotype.Component;
    21
    22 @Component
    23 public class FilePathUtil {
  • Harshit Saxena added 1 commit

    added 1 commit

    Compare with previous version

  • Harshit Saxena added 1 commit

    added 1 commit

    Compare with previous version

  • Harshit Saxena added 1 commit

    added 1 commit

    Compare with previous version

  • Harshit Saxena added 1 commit

    added 1 commit

    Compare with previous version

  • Loading
  • Loading
  • Loading
  • Loading
  • Loading
  • Loading
  • Loading
  • Loading
  • Loading
  • Loading
  • Please register or sign in to reply
    Loading