Merge readmee

README.md

@@ -15,13 +15,13 @@ limitations under the License.
 # Introduction
 
 This repository houses code that is used across the Microsoft Azure hosted OSDU and OpenDES platforms. The intention of this repository is to minimize code duplication for common scenarios such as dependency configuration for services in Azure (KeyVault, Cosmos, Storage and others).
- 
+
 # Pre-requisites
 
 You need
- 
- 1. Maven 3.6.x
- 2. Java 1.8
+
+1. Maven 3.6.x
+2. Java 1.8
 
 # Local Usage
 
@@ -72,7 +72,7 @@ which can be fixed by excluding some of the logger dependencies.
 
 Note: Below are reference PRs for exclusion and might change from service to service
 
-Refer this [MR](https://community.opengroup.org/osdu/platform/security-and-compliance/entitlements-azure/-/merge_requests/13) as reference on how to exclude dependencies along with how to enable the 
+Refer this [MR](https://community.opengroup.org/osdu/platform/security-and-compliance/entitlements-azure/-/merge_requests/13) as reference on how to exclude dependencies along with how to enable the
 Enabled transaction logger and slf4jlogger
 
 ## Environment variables to be added in application.properties to consume the TenantFactoryImpl
@@ -87,3 +87,11 @@ Enabled transaction logger and slf4jlogger
 | ---  | ---   | ---         |
 | `azure.blobStore.required` | `true` | - |
 | `azure.storage.account-name` | ex `testStorage` | storage account name |
+
+# Default retry and timeout values for service-to-service communication
+| name | default value |
+| ---  | ---   | 
+| `maxRetry` | `3` |
+| `connectTimeout` | `60000` |
+| `requestTimeout` | `60000` |
+| `socketTimeout` | `60000` |

src/main/java/org/opengroup/osdu/azure/blobstorage/BlobStore.java

@@ -21,6 +21,7 @@ import com.azure.storage.blob.models.BlobCopyInfo;
 import com.azure.storage.blob.models.BlobErrorCode;
 import com.azure.storage.blob.models.BlobStorageException;
 import com.azure.storage.blob.models.CopyStatusType;
+import com.azure.storage.blob.models.UserDelegationKey;
 import com.azure.storage.blob.sas.BlobContainerSasPermission;
 import com.azure.storage.blob.sas.BlobSasPermission;
 import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
@@ -253,6 +254,8 @@ public class BlobStore {
     }
 
     /**
+     * This method is used to generate pre-signed url for file (blob).
+     * NOTE: Using the below method will require BlobServiceClient to be instantiated using StorageSharedKeyCredential
      * @param dataPartitionId Data partition id
      * @param filePath        Path of file (blob) for which SAS token needs to be generated
      * @param containerName   Name of the storage container
@@ -267,8 +270,8 @@ public class BlobStore {
     }
 
     /**
-     * Generates pre-signed url to a blob container.
-     *
+     * This method is used to generate pre-signed url for blob container.
+     * NOTE: Using the below method will require BlobServiceClient to be instantiated using StorageSharedKeyCredential
      * @param dataPartitionId data partition id
      * @param containerName   Name of the storage container
      * @param expiryTime      Time after which the token expires
@@ -280,6 +283,30 @@ public class BlobStore {
         return blobContainerClient.getBlobContainerUrl() + "?" + generateSASToken(blobContainerClient, expiryTime, permissions);
     }
 
+    /**
+     * Generates pre-signed url to a blob container using the user delegation key.
+     *
+     * @param dataPartitionId data partition id
+     * @param containerName   Name of the storage container
+     * @param startTime       Time after which the token is activated (null in case of instant activation)
+     * @param expiryTime      Time after which the token expires
+     * @param permissions     permissions for the given container
+     * @return Generates pre-signed url for a given container
+     */
+    public String generatePreSignedUrlWithUserDelegationSas(final String dataPartitionId, final String containerName, final OffsetDateTime startTime, final OffsetDateTime expiryTime, final BlobContainerSasPermission permissions) {
+        BlobContainerClient blobContainerClient = getBlobContainerClient(dataPartitionId, containerName);
+        BlobServiceClient blobServiceClient = blobServiceClientFactory.getBlobServiceClient(dataPartitionId);
+        UserDelegationKey userDelegationKey = blobServiceClient.getUserDelegationKey(startTime, expiryTime);
+        BlobServiceSasSignatureValues blobServiceSasSignatureValues = new BlobServiceSasSignatureValues(expiryTime, permissions).setStartTime(startTime);
+
+        final long start = System.currentTimeMillis();
+        String sasToken = blobContainerClient.generateUserDelegationSas(blobServiceSasSignatureValues, userDelegationKey);
+        final long timeTaken = System.currentTimeMillis() - start;
+
+        logDependency("GENERATE_PRESIGNED_URL_USER_DELEGATION_SAS", blobContainerClient.getBlobContainerName(), blobContainerClient.getBlobContainerUrl(), timeTaken, String.valueOf(HttpStatus.SC_OK), true);
+        return blobContainerClient.getBlobContainerUrl() + "?" + sasToken;
+    }
+
     /**
      * Method is used to copy a file specified at Source URL to the provided destination.
      *

src/main/java/org/opengroup/osdu/azure/di/RetryAndTimeoutConfiguration.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
+package org.opengroup.osdu.azure.di;
+
+import lombok.Data;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+/**
+ * Configuration bean for setting up retry and timeouts variables.
+ */
+
+@Data
+@Component
+@ConfigurationProperties(prefix = "azure.service.retry.config")
+public class RetryAndTimeoutConfiguration {
+
+    private int maxRetry = 3;
+    private int connectTimeout = 60000;
+    private int requestTimeout = 60000;
+    private int socketTimeout = 60000;
+}

src/main/java/org/opengroup/osdu/azure/entitlements/EntilementsAPIConfigBean.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
+package org.opengroup.osdu.azure.entitlements;
+
+import lombok.Data;
+import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
+import org.springframework.beans.factory.config.AbstractFactoryBean;
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.stereotype.Component;
+
+/**
+ * Creates bean of EntitlementsAPIConfig.
+ */
+@Data
+@Component
+@ConfigurationProperties
+public class EntilementsAPIConfigBean extends AbstractFactoryBean<EntitlementsAPIConfig> {
+
+    private String authorizeAPI;
+
+    private String authorizeAPIKey;
+
+    /**
+     * Abstract method of AbstractBeanFactory<T>.
+     *
+     * @return class type
+     */
+    @Override
+    public Class<?> getObjectType() {
+        return EntitlementsAPIConfig.class;
+    }
+
+    /**
+     * Abstract method of AbstractBeanFactory<T> type.
+     *
+     * @return EntitlementsAPIConfig
+     * @throws Exception
+     */
+    @Override
+    protected EntitlementsAPIConfig createInstance() throws Exception {
+        return EntitlementsAPIConfig
+                .builder()
+                .rootUrl(authorizeAPI)
+                .apiKey(authorizeAPIKey)
+                .build();
+    }
+}

src/main/java/org/opengroup/osdu/azure/entitlements/EntitlementsFactoryAzure.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
+package org.opengroup.osdu.azure.entitlements;
+
+import org.opengroup.osdu.core.common.entitlements.EntitlementsAPIConfig;
+import org.opengroup.osdu.core.common.entitlements.EntitlementsService;
+import org.opengroup.osdu.core.common.entitlements.IEntitlementsFactory;
+import org.opengroup.osdu.core.common.entitlements.IEntitlementsService;
+import org.opengroup.osdu.core.common.http.IHttpClient;
+import org.opengroup.osdu.core.common.http.json.HttpResponseBodyMapper;
+import org.opengroup.osdu.core.common.model.http.DpsHeaders;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+import java.util.Objects;
+
+/**
+ * Implements IEntitlementsFactory.
+ */
+@Component
+@Primary
+@ConditionalOnProperty(value = "azure.entitlements.factory.enabled", havingValue = "true", matchIfMissing = true)
+public class EntitlementsFactoryAzure implements IEntitlementsFactory {
+
+    private final EntitlementsAPIConfig config;
+    private final HttpResponseBodyMapper mapper;
+    private final IHttpClient client;
+
+    /**
+     * Constructor Injection for above 3 fields.
+     *
+     * @param entitlementsConfig EntitlementsAPIConfig
+     * @param httpMapper HttpResponseBodyMapper
+     * @param httpClient IHttpClient
+     */
+    @Autowired
+    public EntitlementsFactoryAzure(final EntitlementsAPIConfig entitlementsConfig, final HttpResponseBodyMapper httpMapper, final IHttpClient httpClient) {
+        this.config = entitlementsConfig;
+        this.mapper = httpMapper;
+        this.client = httpClient;
+    }
+
+    /**
+     * returns instance of EntitlementsService.
+     *
+     * @param headers DpsHeaders
+     * @return IEntitlementsService
+     */
+    @Override
+    public IEntitlementsService create(final DpsHeaders headers) {
+        Objects.requireNonNull(headers, "headers cannot be null");
+
+        return new EntitlementsService(this.config,
+                this.client,
+                headers,
+                this.mapper);
+    }
+}

src/main/java/org/opengroup/osdu/azure/httpconfig/HttpClientAzure.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
+package org.opengroup.osdu.azure.httpconfig;
+
+import org.apache.http.HttpStatus;
+import org.opengroup.osdu.core.common.http.FetchServiceHttpRequest;
+import org.opengroup.osdu.core.common.http.HttpRequest;
+import org.opengroup.osdu.core.common.http.IHttpClient;
+import org.opengroup.osdu.core.common.http.HttpResponse;
+import org.opengroup.osdu.core.common.http.UrlFetchServiceImpl;
+import org.opengroup.osdu.core.common.model.http.AppException;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+import java.net.URISyntaxException;
+
+/**
+ * Extends URlFetchService and implements IHttpClient to send requests.
+ */
+@Primary
+@Component
+public class HttpClientAzure extends UrlFetchServiceImpl implements IHttpClient {
+
+    /**
+     * calls urlfetchservice's send request.
+     *
+     * @param httpRequest made by user class
+     * @return HttpResponse
+     */
+    @Override
+    public HttpResponse send(final HttpRequest httpRequest) {
+        org.opengroup.osdu.core.common.model.http.HttpResponse response = null;
+        try {
+            response = super.sendRequest(FetchServiceHttpRequest.builder()
+                    .body(httpRequest.getBody())
+                    .httpMethod(httpRequest.getHttpMethod())
+                    .queryParams(httpRequest.getQueryParams())
+                    .url(httpRequest.getUrl())
+                    .headers(httpRequest.getHeaders())
+                    .build());
+        } catch (URISyntaxException e) {
+            throw new AppException(HttpStatus.SC_INTERNAL_SERVER_ERROR, e.getReason(), "URI Syntax is not correct", e);
+        }
+        HttpResponse httpResponse = new HttpResponse();
+        httpResponse.setBody(response.getBody());
+        httpResponse.setResponseCode(response.getResponseCode());
+        httpResponse.setContentType(response.getContentType());
+        httpResponse.setRequest(httpRequest);
+        return httpResponse;
+    }
+}

src/main/java/org/opengroup/osdu/azure/httpconfig/HttpClientHandlerAzure.java

+// Copyright © Microsoft Corporation
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License
+
+package org.opengroup.osdu.azure.httpconfig;
+
+import org.apache.http.client.config.RequestConfig;
+import org.opengroup.osdu.azure.di.RetryAndTimeoutConfiguration;
+import org.opengroup.osdu.core.common.http.HttpClientHandler;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.annotation.Primary;
+import org.springframework.stereotype.Component;
+
+/**
+ * Extends HttpClientHandler.
+ */
+@Primary
+@Component
+public class HttpClientHandlerAzure extends HttpClientHandler {
+
+    private RetryAndTimeoutConfiguration configuration;
+
+    /**
+     * Constuctor injection for RetryAndTimeoutConfiguration.
+     *
+     * @param retryAndTimeoutConfiguration of type RetryAndTimeoutConfiguration
+     */
+    @Autowired
+    public HttpClientHandlerAzure(final RetryAndTimeoutConfiguration retryAndTimeoutConfiguration) {
+        this.configuration = retryAndTimeoutConfiguration;
+        super.REQUEST_CONFIG = RequestConfig.custom()
+                .setConnectTimeout(configuration.getConnectTimeout())
+                .setConnectionRequestTimeout(configuration.getRequestTimeout())
+                .setSocketTimeout(configuration.getSocketTimeout()).build();
+
+        super.RETRY_COUNT = configuration.getMaxRetry();
+    }
+
+}

src/main/java/org/opengroup/osdu/azure/partition/PartitionInfoAzure.java

@@ -58,6 +58,12 @@ public class PartitionInfoAzure {
     @SerializedName("storage-account-name")
     private Property storageAccountNameConfig;
 
+    @SerializedName("ingest-storage-account-key")
+    private Property ingestStorageAccountKeyConfig;
+
+    @SerializedName("ingest-storage-account-name")
+    private Property ingestStorageAccountNameConfig;
+
     @SerializedName("sb-namespace")
     private Property sbNamespaceConfig;
 
@@ -230,6 +236,26 @@ public class PartitionInfoAzure {
         return String.valueOf(this.getStorageAccountNameConfig().getValue());
     }
 
+    /**
+     * @return ingestion storage account key
+     */
+    public String getIngestStorageAccountKey() {
+        if (this.getIngestStorageAccountKeyConfig().isSensitive()) {
+            return getSecret(this.getIngestStorageAccountKeyConfig());
+        }
+        return String.valueOf(this.getIngestStorageAccountKeyConfig().getValue());
+    }
+
+    /**
+     * @return ingestion storage account name
+     */
+    public String getIngestStorageAccountName() {
+        if (this.getIngestStorageAccountNameConfig().isSensitive()) {
+            return getSecret(this.getIngestStorageAccountNameConfig());
+        }
+        return String.valueOf(this.getIngestStorageAccountNameConfig().getValue());
+    }
+
     /**
      * @return partition event grid topic endpoint
      */

src/test/java/org/opengroup/osdu/azure/blobstorage/BlobStoreTest.java

@@ -24,6 +24,7 @@ import com.azure.storage.blob.sas.BlobContainerSasPermission;
 import com.azure.storage.blob.sas.BlobSasPermission;
 import com.azure.storage.blob.sas.BlobServiceSasSignatureValues;
 import com.azure.storage.blob.specialized.BlockBlobClient;
+import org.apache.catalina.User;
 import org.junit.jupiter.api.AfterEach;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
@@ -423,6 +424,45 @@ public class BlobStoreTest {
         assertEquals(containerPreSignedUrl, obtainedPreSignedUrl);
     }
 
+    @Test
+    public void generatePreSignedUrlWithUserDelegationSas_NullPreSignedTokenObtained() {
+        int expiryDays = 1;
+        OffsetDateTime startTime = OffsetDateTime.now();
+        OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays);
+        BlobContainerSasPermission blobContainerSasPermission = (new BlobContainerSasPermission()).setReadPermission(true).setCreatePermission(true);
+        String obtainedPreSignedUrl = blobStore.generatePreSignedUrlWithUserDelegationSas(PARTITION_ID, STORAGE_CONTAINER_NAME, startTime, expiryTime, blobContainerSasPermission);
+        assertEquals("null?null", obtainedPreSignedUrl);
+    }
+
+    @Test
+    public void generatePreSignedURLlWithUserDelegationSas_whenContainerPreSignedUrl_thenReturnsValidSasToken() {
+        UserDelegationKey userDelegationKey = mock(UserDelegationKey.class);
+
+        String containerSasToken = "containerSasToken";
+        String containerUrl = "containerUrl";
+        String containerPreSignedUrl = containerUrl + "?" + containerSasToken;
+
+        doReturn(userDelegationKey).when(blobServiceClient).getUserDelegationKey(any(OffsetDateTime.class), any(OffsetDateTime.class));
+        doReturn(containerUrl).when(blobContainerClient).getBlobContainerUrl();
+        doReturn(containerSasToken).when(blobContainerClient).generateUserDelegationSas(any(BlobServiceSasSignatureValues.class), any(UserDelegationKey.class));
+
+        int expiryDays = 1;
+        OffsetDateTime startTime = OffsetDateTime.now();
+        OffsetDateTime expiryTime = OffsetDateTime.now().plusDays(expiryDays);
+        BlobContainerSasPermission blobContainerSasPermission = (new BlobContainerSasPermission()).setReadPermission(true).setCreatePermission(true);
+        String obtainedPreSignedUrl = blobStore.generatePreSignedUrlWithUserDelegationSas(PARTITION_ID, STORAGE_CONTAINER_NAME, startTime, expiryTime, blobContainerSasPermission);
+
+        ArgumentCaptor<BlobServiceSasSignatureValues> blobServiceSasSignatureValuesArgumentCaptor = ArgumentCaptor.forClass(BlobServiceSasSignatureValues.class);
+        ArgumentCaptor<UserDelegationKey> userDelegationKeyArgumentCaptor = ArgumentCaptor.forClass(UserDelegationKey.class);
+        verify(blobContainerClient).generateUserDelegationSas(blobServiceSasSignatureValuesArgumentCaptor.capture(), userDelegationKeyArgumentCaptor.capture());
+
+        assertEquals(blobContainerSasPermission.toString(), blobServiceSasSignatureValuesArgumentCaptor.getValue().getPermissions());
+        assertEquals(userDelegationKey, userDelegationKeyArgumentCaptor.getValue());
+        assertEquals(startTime, blobServiceSasSignatureValuesArgumentCaptor.getValue().getStartTime());
+        assertEquals(expiryTime, blobServiceSasSignatureValuesArgumentCaptor.getValue().getExpiryTime());
+        assertEquals(containerPreSignedUrl, obtainedPreSignedUrl);
+    }
+
     private BlobStorageException mockStorageException(BlobErrorCode errorCode) {
         BlobStorageException mockException = mock(BlobStorageException.class);
         lenient().when(mockException.getErrorCode()).thenReturn(errorCode);