Merge branch 'trusted-updated-vendors-loading' into 'master'

GONRG-1866: add condition statement to import vendors dependencies

See merge request !27

src/dags/providers/README.md

@@ -12,7 +12,7 @@ providers
     │   ...
 ```
 In the base folder there are the following base classes:
-`types.py`: Stores all the interfaces to be implemented by all cloud providers.
+`types.py`: Stores all the interfaces that could be implemented by cloud providers.
 `factory.py` Provides a mechanism to register and retrieve cloud specific implementations using a class decorator @ProviderFactory.register. A registry per interface is required, if a new interface is implemented a new registry should be added.
 `constants.py` Reusable constants.
 `exceptions.py` Provider specific exceptions should be thrown here and bubble up all the way up.

src/dags/providers/blob_storage.py

@@ -14,17 +14,27 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """Blob storage init module."""
-
+import importlib
 import os
 
+import logging
+
 from providers.factory import ProvidersFactory
-# import section needed to register cloud specific clients
-from providers.azure import azure_blob_storage_client
-from providers.ibm import ibm_blob_storage_client
-from providers.gcp import gcp_blob_storage_client  # pylint: disable=unused-import
 from providers.types import BlobStorageClient
 
 
+logger = logging.getLogger()
+
+
+def _import_provider_specific_storage_client_module(provider: str) -> str:
+    """
+    Import provider specific blob storage client module for correct registering it
+    """
+    module_name = f"providers.{provider.lower()}.{provider.lower()}_blob_storage_client"
+    importlib.import_module(module_name)
+    return module_name
+
+
 def get_client(cloud_env: str = None) -> BlobStorageClient:
     """Get specific blob storage client according to cloud environment.
 
@@ -35,4 +45,10 @@ def get_client(cloud_env: str = None) -> BlobStorageClient:
     :rtype: BlobStorageClient
     """
     cloud_env = cloud_env or os.environ.get("CLOUD_PROVIDER")
+    # import section needed to register cloud specific clients
+    try:
+        _import_provider_specific_storage_client_module(cloud_env)
+    except ModuleNotFoundError as exc:
+        logger.critical(f"Error occurred while importing blob storage client module for {cloud_env}")
+        logger.critical(f"Exception: {exc}")
     return ProvidersFactory.get_blob_storage_client(cloud_env)

src/dags/providers/credentials.py

@@ -14,17 +14,27 @@
 #  See the License for the specific language governing permissions and
 #  limitations under the License.
 """Credentials init module."""
-
+import importlib
 import os
 
+import logging
+
 from providers.factory import ProvidersFactory
-# import section needed to register cloud specific clients
-from providers.azure import azure_credentials
-from providers.ibm import ibm_credentials
-from providers.gcp import gcp_credentials  # pylint: disable=unused-import
 from providers.types import BaseCredentials
 
 
+logger = logging.getLogger()
+
+
+def _import_provider_specific_credential_module(provider: str) -> str:
+    """
+    Import provider specific credential module for correct registering it
+    """
+    module_name = f"providers.{provider.lower()}.{provider.lower()}_credentials"
+    importlib.import_module(module_name)
+    return module_name
+
+
 def get_credentials(cloud_env: str = None) -> BaseCredentials:
     """Get specific Credentials according to cloud environment.
 
@@ -35,4 +45,10 @@ def get_credentials(cloud_env: str = None) -> BaseCredentials:
     :rtype: BaseCredentials
     """
     cloud_env = cloud_env or os.environ.get("CLOUD_PROVIDER")
+    # import section needed to register cloud specific clients
+    try:
+        _import_provider_specific_credential_module(cloud_env)
+    except ModuleNotFoundError as exc:
+        logger.critical(f"Error occurred while importing credential module for {cloud_env}")
+        logger.critical(f"Exception: {exc}")
     return ProvidersFactory.get_credentials(cloud_env)

src/dags/providers/exceptions.py

@@ -15,6 +15,7 @@
 
 """Providers exceptions module."""
 
+
 class RefreshSATokenError(Exception):
     """Raise when token is empty after attempt to get credentials from service account file."""
     pass

tests/providers-unit-tests/test_blob_storage_client.py

@@ -22,8 +22,10 @@ sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/dags")
 import pytest
 import providers.blob_storage
 from providers.blob_storage import get_client
+from providers.factory import ProvidersFactory
 from providers.types import BlobStorageClient
 from providers.gcp.gcp_blob_storage_client import GoogleCloudStorageClient
+from providers.azure.azure_blob_storage_client import AzureCloudStorageClient
 
 DATA_PATH_PREFIX = f"{os.path.dirname(__file__)}/data"
 
@@ -57,3 +59,12 @@ class TestGetBlobStorageClient:
         """
         with pytest.raises(NotImplementedError):
             get_client(provider)
+
+    @pytest.mark.parametrize("provider", [
+        pytest.param("gcp"),
+        pytest.param("azure"),
+    ])
+    def test_verify_registries(self, monkeypatch, mock_os_environ, provider: str):
+        """Test import actually registered in factory
+        """
+        assert ProvidersFactory.blob_storage_registry.get(provider) is not None

tests/providers-unit-tests/test_credentials.py

@@ -22,6 +22,7 @@ sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/dags")
 import pytest
 import providers.credentials
 from providers.credentials import get_credentials
+from providers.factory import ProvidersFactory
 from providers.types import BaseCredentials
 from providers.gcp.gcp_credentials import GCPCredentials
 from providers.azure.azure_credentials import AzureCredentials
@@ -70,3 +71,12 @@ class TestGetCredentials:
         """
         with pytest.raises(NotImplementedError):
             get_credentials(provider)
+
+    @pytest.mark.parametrize("provider", [
+        pytest.param("gcp"),
+        pytest.param("azure")
+    ])
+    def test_verify_registries(self, monkeypatch, mock_os_environ, provider: str):
+        """Test import actually registered in factory
+        """
+        assert ProvidersFactory.credentials_registry.get(provider) is not None

tests/providers-unit-tests/test_provider_specific_imports.py

+#  Copyright 2021 Google LLC
+#  Copyright 2021 EPAM Systems
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+import importlib
+import os
+import sys
+
+sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/plugins")
+sys.path.append(f"{os.getenv('AIRFLOW_SRC_DIR')}/dags")
+
+import pytest
+from providers.credentials import _import_provider_specific_credential_module
+from providers.blob_storage import _import_provider_specific_storage_client_module
+
+
+DATA_PATH_PREFIX = f"{os.path.dirname(__file__)}/data"
+
+
+class TestCloudSpecificImport:
+    """Test providers modules can be import successfully."""
+    @pytest.fixture()
+    def mock_os_environ(self, monkeypatch, provider: str):
+        """Mock os environ with a custom provider.
+        """
+        monkeypatch.setattr(
+            os, "environ", {
+                "CLOUD_PROVIDER": provider,
+                "GOOGLE_APPLICATION_CREDENTIALS": f"{DATA_PATH_PREFIX}/fake_sa_file.json"
+            })
+
+    @pytest.mark.parametrize("provider, module_name", [
+        pytest.param("gcp", "providers.gcp.gcp_credentials"),
+        pytest.param("azure", "providers.azure.azure_credentials"),
+        pytest.param("ibm", "providers.ibm.ibm_credentials"),
+    ])
+    def test_import_credentials_module(self, monkeypatch, mock_os_environ,
+                                       provider: str, module_name: str):
+        """Test import credential module.
+        """
+        assert _import_provider_specific_credential_module(provider) == module_name
+
+    @pytest.mark.parametrize("provider", [
+        pytest.param("other"),
+    ])
+    def test_import_credentials_module_raises(self, monkeypatch, mock_os_environ, provider: str):
+        """Test import credential module raises error if module doesn't exist.
+        """
+        with pytest.raises(ModuleNotFoundError):
+            _import_provider_specific_credential_module(provider)
+
+    @pytest.mark.parametrize("provider, module_name", [
+        pytest.param("gcp", "providers.gcp.gcp_blob_storage_client"),
+        pytest.param("azure", "providers.azure.azure_blob_storage_client"),
+        pytest.param("ibm", "providers.ibm.ibm_blob_storage_client"),
+    ])
+    def test_import_blob_storage_client_module(self, monkeypatch, mock_os_environ, provider: str, module_name: str):
+        """Test import blob storage client module.
+        """
+        assert _import_provider_specific_storage_client_module(provider) == module_name
+
+    @pytest.mark.parametrize("provider", [
+        pytest.param("other"),
+    ])
+    def test_import_blob_storage_client_module_raises(self, monkeypatch, mock_os_environ, provider: str):
+        """Test import blob storage client module raises error if module doesn't exist.
+        """
+        with pytest.raises(ModuleNotFoundError):
+            _import_provider_specific_storage_client_module(provider)