Merge branch 'Azure/OSDU-Wellbore-DDMS-M8-Master' into 'azure/m10-master'

Merge from M8-Azure to m10-master to onboard wellbore to latest OSDU version

See merge request !498

.gitlab-ci.yml

@@ -43,37 +43,37 @@ variables:
 include:
   - project: "osdu/platform/ci-cd-pipelines"
     file: "standard-setup.yml"
-    ref: v0.13.2
+    ref: v0.13.3
 
   - project: "osdu/platform/ci-cd-pipelines"
     file: "build/python.yml"
-    ref: v0.13.2
+    ref: v0.13.3
 
   - project: "osdu/platform/ci-cd-pipelines"
     file: "scanners/fossa-python.yml"
-    ref: v0.13.2
+    ref: v0.13.3
 
   - project: "osdu/platform/ci-cd-pipelines"
     file: "scanners/gitlab-ultimate.yml"
-    ref: v0.13.2
+    ref: v0.13.3
 
   - project: "osdu/platform/ci-cd-pipelines"
     file: "cloud-providers/osdu-gcp-gke.yml"
-    ref: v0.13.2
+    ref: v0.13.3
 
   - project: "osdu/platform/ci-cd-pipelines"
     file: "cloud-providers/ibm-wellbore-git.yml"
-    ref: v0.13.2
+    ref: v0.13.3
 
   - local: "/devops/azure/azure-wellbore.yml"
 
   - project: "osdu/platform/ci-cd-pipelines"
     file: "cloud-providers/aws-global.yml"
-    ref: v0.13.2
+    ref: v0.13.3
 
   - project: "osdu/platform/ci-cd-pipelines"
     file: "cloud-providers/aws-python.yml"
-    ref: v0.13.2
+    ref: v0.13.3
 
 verify_existing_requirements:
   extends:

NOTICE

@@ -20,9 +20,7 @@ The following software have components provided under the terms of this license:
 - google-api-core (from https://github.com/GoogleCloudPlatform/google-cloud-python)
 - google-auth (from https://github.com/GoogleCloudPlatform/google-auth-library-python, https://github.com/googleapis/google-auth-library-python)
 - google-auth-oauthlib (from https://github.com/GoogleCloudPlatform/google-auth-library-python-oauthlib)
-- google-cloud-core (from https://github.com/GoogleCloudPlatform/google-cloud-python)
-- google-cloud-monitoring (from https://github.com/GoogleCloudPlatform/google-cloud-python)
-- google-cloud-trace (from https://github.com/googleapis/googleapis)
+- google-cloud-core (from https://github.com/googleapis/python-cloud-core)
 - googleapis-common-protos (from https://github.com/googleapis/googleapis)
 - grpcio (from https://grpc.io)
 - importlib-metadata
@@ -49,10 +47,8 @@ The following software have components provided under the terms of this license:
 - requests (from http://python-requests.org, https://requests.readthedocs.io)
 - rfc3986 (from https://rfc3986.readthedocs.org)
 - rsa (from https://stuvel.eu/rsa)
-- s3transfer (from https://github.com/boto/s3transfer)
 - sniffio (from https://github.com/python-trio/sniffio)
 - sortedcontainers (from http://www.grantjenks.com/docs/sortedcontainers/)
-- structlog (from http://www.structlog.org/)
 - tblib (from https://github.com/ionelmc/python-tblib)
 - toposort (from https://bitbucket.org/ericvsmith/toposort)
 - tornado (from http://www.tornadoweb.org/)
@@ -92,9 +88,6 @@ The following software have components provided under the terms of this license:
 - cryptography (from https://github.com/pyca/cryptography)
 - dask (from http://github.com/dask/dask/)
 - decorator (from https://github.com/micheles/decorator)
-- distributed (from https://distributed.readthedocs.io/en/latest/)
-- fsspec (from http://github.com/intake/filesystem_spec)
-- gcsfs (from https://github.com/dask/gcsfs)
 - grpcio (from https://grpc.io)
 - hiredis (from https://github.com/redis/hiredis-py)
 - httpcore (from https://github.com/encode/httpcore)
@@ -117,6 +110,7 @@ The following software have components provided under the terms of this license:
 - pyasn1 (from http://sourceforge.net/projects/pyasn1/)
 - pyasn1-modules (from http://sourceforge.net/projects/pyasn1/)
 - pycparser (from https://github.com/eliben/pycparser)
+- pyparsing (from http://pyparsing.wikispaces.com/)
 - pyrsistent (from http://github.com/tobgu/pyrsistent/)
 - python-dateutil (from https://dateutil.readthedocs.org)
 - python-rapidjson (from https://github.com/python-rapidjson/python-rapidjson)
@@ -168,11 +162,11 @@ The following software have components provided under the terms of this license:
 - toolz (from http://github.com/pytoolz/toolz/)
 
 ========================================================================
-CC-BY-SA-3.0
+CC0-1.0
 ========================================================================
 The following software have components provided under the terms of this license:
 
-- numpy
+- pandas (from https://pandas.pydata.org)
 
 ========================================================================
 GPL-2.0-only
@@ -197,7 +191,6 @@ The following software have components provided under the terms of this license:
 - Pillow (from https://python-pillow.org)
 - coverage (from https://github.com/nedbat/coveragepy)
 - grpcio (from https://grpc.io)
-- pyparsing (from http://pyparsing.wikispaces.com/)
 - rfc3986 (from https://rfc3986.readthedocs.org)
 
 ========================================================================
@@ -213,7 +206,6 @@ ISC
 The following software have components provided under the terms of this license:
 
 - grpcio (from https://grpc.io)
-- requests-oauthlib (from https://github.com/requests/requests-oauthlib)
 
 ========================================================================
 Info-ZIP
@@ -222,13 +214,6 @@ The following software have components provided under the terms of this license:
 
 - grpcio (from https://grpc.io)
 
-========================================================================
-JSON
-========================================================================
-The following software have components provided under the terms of this license:
-
-- python-rapidjson (from https://github.com/python-rapidjson/python-rapidjson)
-
 ========================================================================
 LGPL-2.1-only
 ========================================================================
@@ -236,7 +221,6 @@ The following software have components provided under the terms of this license:
 
 - Pillow (from https://python-pillow.org)
 - chardet (from https://github.com/chardet/chardet)
-- charset-normalizer (from https://github.com/ousret/charset_normalizer)
 
 ========================================================================
 LGPL-2.1-or-later
@@ -261,7 +245,7 @@ The following software have components provided under the terms of this license:
 
 - Pillow (from https://python-pillow.org)
 - PyJWT (from http://github.com/jpadilla/pyjwt)
-- PyYAML (from http://pyyaml.org/wiki/PyYAML)
+- PyYAML (from )
 - adal (from https://github.com/AzureAD/azure-activedirectory-library-for-python)
 - aiohttp (from https://github.com/aio-libs/aiohttp, https://github.com/aio-libs/aiohttp/)
 - aioitertools (from https://github.com/jreese/aioitertools)
@@ -272,11 +256,6 @@ The following software have components provided under the terms of this license:
 - azure-common (from https://github.com/Azure/azure-sdk-for-python)
 - azure-core (from https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/core/azure-core)
 - azure-datalake-store (from https://github.com/Azure/azure-data-lake-store-python)
-- azure-identity (from https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/identity/azure-identity)
-- azure-keyvault (from https://github.com/Azure/azure-sdk-for-python)
-- azure-keyvault-certificates (from https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/keyvault/azure-keyvault-certificates)
-- azure-keyvault-keys (from https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/keyvault/azure-keyvault-keys)
-- azure-keyvault-secrets (from https://github.com/Azure/azure-sdk-for-python/tree/master/sdk/keyvault/azure-keyvault-secrets)
 - azure-storage-blob (from https://github.com/Azure/azure-storage-python)
 - backoff
 - bokeh (from https://github.com/bokeh/bokeh)
@@ -294,9 +273,7 @@ The following software have components provided under the terms of this license:
 - jsonschema
 - mockito (from https://github.com/kaste/mockito-python)
 - msal (from https://github.com/AzureAD/microsoft-authentication-library-for-python)
-- msal-extensions (from https://pypi.org/project/msal-extensions/0.1.3/)
 - msrest (from https://github.com/Azure/msrest-for-python)
-- munch (from http://github.com/Infinidat/munch)
 - natsort (from https://github.com/SethMMorton/natsort)
 - numpy
 - ordered-set (from http://github.com/LuminosoInsight/ordered-set)
@@ -305,25 +282,24 @@ The following software have components provided under the terms of this license:
 - pluggy
 - py (from http://pylib.readthedocs.org/)
 - pyarrow (from https://arrow.apache.org/)
-- pydantic (from https://github.com/samuelcolvin/pydantic)
 - pyparsing (from http://pyparsing.wikispaces.com/)
 - pyrsistent (from http://github.com/tobgu/pyrsistent/)
-- pytest (from http://pytest.org)
+- pytest (from https://docs.pytest.org/en/latest/)
 - pytest-cov (from https://github.com/pytest-dev/pytest-cov)
-- pytest-httpx (from https://colin-b.github.io/pytest_httpx/)
 - pytest-mock (from https://github.com/pytest-dev/pytest-mock/)
-- python-rapidjson (from https://github.com/python-rapidjson/python-rapidjson)
-- python-ulid (from https://github.com/mdomke/python-ulid)
 - pytz (from http://pythonhosted.org/pytz)
-- requests-oauthlib (from https://github.com/requests/requests-oauthlib)
 - six (from http://pypi.python.org/pypi/six/)
 - sniffio (from https://github.com/python-trio/sniffio)
-- structlog (from http://www.structlog.org/)
 - toml (from https://github.com/uiri/toml)
 - tomli (from https://pypi.org/project/tomli/1.2.2/, https://pypi.org/project/tomli/2.0.0/)
 - urllib3 (from https://urllib3.readthedocs.io/)
-- xmltodict (from https://github.com/martinblech/xmltodict)
-- zipp (from https://github.com/jaraco/zipp)
+
+========================================================================
+MIT-CMU
+========================================================================
+The following software have components provided under the terms of this license:
+
+- pyparsing (from http://pyparsing.wikispaces.com/)
 
 ========================================================================
 MPL-2.0
@@ -331,7 +307,6 @@ MPL-2.0
 The following software have components provided under the terms of this license:
 
 - certifi (from http://certifi.io/)
-- charset-normalizer (from https://github.com/ousret/charset_normalizer)
 
 ========================================================================
 NCSA
@@ -390,11 +365,11 @@ The following software have components provided under the terms of this license:
 - numpy
 
 ========================================================================
-Unlicense
+TCL
 ========================================================================
 The following software have components provided under the terms of this license:
 
-- grpcio (from https://grpc.io)
+- pandas (from https://pandas.pydata.org)
 
 ========================================================================
 WTFPL

README.md

@@ -321,7 +321,7 @@ Then access app on `http://127.0.0.1:<LOCAL_PORT>/api/os-wellbore-ddms/docs`
 IMAGE_TAG="os-wellbore-ddms:dev"
 
 # Build Image
-docker build -t=$IMAGE_TAG --rm . -f ./build/dockerfile --build-arg PIP_WHEEL_DIR=python-packages
+docker build -t=$IMAGE_TAG --rm . -f ./build/Dockerfile --build-arg PIP_WHEEL_DIR=python-packages
 ```
 
 #### Run Image

app/conf.py

@@ -305,7 +305,7 @@ def cloud_provider_additional_environment(config: ConfigurationContainer):
                             env_var_key='AZ_AI_INSTRUMENTATION_KEY',
                             description='azure app insights instrumentation key',
                             secret=True,
-                            is_mandatory=True,
+                            is_mandatory=False,
                             override=True)
 
         config.add_from_env(attribute_name='az_logger_level',

app/helper/logger.py

@@ -118,27 +118,22 @@ def create_azure_logger(service_name):
 
     #  AzurelogHandler for logging to azure appinsight
     key = Config.get('az_ai_instrumentation_key')
-    logger_level = Config.get('az_logger_level')
-    az_handler = AzureLogHandler(connection_string=f'InstrumentationKey={key}')
-    az_handler.setLevel(logging.getLevelName(logger_level))
-    az_handler.add_telemetry_processor(rename_cloud_role_func(service_name))
+    if key:
+        logger_level = Config.get('az_logger_level')
+        az_handler = AzureLogHandler(connection_string=f'InstrumentationKey={key}')
+        az_handler.setLevel(logging.getLevelName(logger_level))
+        az_handler.add_telemetry_processor(rename_cloud_role_func(service_name))
+    else:
+        az_handler = None
 
     # Acquire the logger for azure library
-    az_logger = logging.getLogger('azure')
-    az_logger.setLevel(logging.INFO)
-    az_logger.addHandler(stdout_handler)
+    az_logger = __getLogger(logger_name='azure', log_level=logging.DEBUG, stdout_handler=stdout_handler, azure_handler=az_handler)
 
     # Acquire the logger for osdu-core-lib-python-azure
-    osdu_core_lib_logger = logging.getLogger('osdu_az')
-    osdu_core_lib_logger.setLevel(logging.INFO)
-    osdu_core_lib_logger.addHandler(stdout_handler)
-    osdu_core_lib_logger.addHandler(az_handler)
+    osdu_core_lib_logger = __getLogger(logger_name='osdu_az', log_level=logging.DEBUG, stdout_handler=stdout_handler, azure_handler=az_handler)
 
     # Acquire the logger for wdms
-    logger = logging.getLogger(__name__)
-    logger.setLevel(logging.INFO)
-    logger.addHandler(stdout_handler)
-    logger.addHandler(az_handler)
+    logger = __getLogger(logger_name=__name__, log_level=logging.DEBUG, stdout_handler=stdout_handler, azure_handler=az_handler)
 
     # return wdms logger with Context adapter
     return AzureContextLoggerAdapter(logger, extra=dict())
@@ -183,3 +178,13 @@ def create_gcp_logger(service_name):
     std_ddms_app.propagate = False
 
     return my_logger
+
+def __getLogger(logger_name, log_level, stdout_handler, azure_handler=None):
+    logger = logging.getLogger(logger_name)
+    logger.setLevel(log_level)
+    logger.addHandler(stdout_handler)
+
+    if azure_handler:
+        logger.addHandler(azure_handler)
+
+    return logger
\ No newline at end of file

app/helper/metric.py

+from app.conf import Config
+from prometheus_fastapi_instrumentator import Instrumentator
+
+def init_metric(wdms_app):
+
+    if Config.cloud_provider.value == 'az':
+         Instrumentator().instrument(wdms_app).expose(wdms_app)
\ No newline at end of file

app/helper/traces.py

@@ -92,9 +92,10 @@ def create_exporter(service_name):
 
         key = Config.get('az_ai_instrumentation_key')
         try:
-            az_exporter = _create_azure_exporter(key)
-            az_exporter.add_telemetry_processor(rename_cloud_role_func(service_name))
-            combined_exporter.add_exporter(az_exporter)
+            if type(key) is not None:
+                az_exporter = _create_azure_exporter(key)
+                az_exporter.add_telemetry_processor(rename_cloud_role_func(service_name))
+                combined_exporter.add_exporter(az_exporter)
         except ValueError as e:
             print('Unable to create AzureExporter:', str(e))
     else:

app/wdms_app.py

@@ -27,7 +27,7 @@ from app.helper.traces import TracingRoute
 from app.model.entity_utils import Entity
 from app.modules import discoverer
 
-from app.helper import traces, logger
+from app.helper import traces, logger, metric
 from app.injector.app_injector import AppInjector
 from app.injector.main_injector import MainInjector
 from app.middleware import CreateBasicContextMiddleware, TracingMiddleware
@@ -147,6 +147,7 @@ async def startup_event():
         asyncio.create_task(run_in_pool_executor(executor_startup_task))
 
     add_modules_routers()
+    metric.init_metric(wdms_app)
 
 
 @base_app.on_event('shutdown')

requirements.in

@@ -12,6 +12,7 @@ python-multipart
 jsonpath-ng # TODO: remove when bulk_path no longer supported
 jsonbender==0.9.3
 natsort
+prometheus-fastapi-instrumentator
 
 opencensus
 opencensus-ext-stackdriver

requirements.txt

 #
-# This file is autogenerated by pip-compile with python 3.7
+# This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -135,6 +135,7 @@ fastapi==0.68.1
     #   -r requirements.in
     #   osdu-data-ecosystem-search
     #   osdu-data-ecosystem-storage
+    #   prometheus-fastapi-instrumentator
 fsspec==2021.7.0
     # via
     #   -r requirements.in
@@ -192,10 +193,6 @@ idna==3.2
     #   requests
     #   rfc3986
     #   yarl
-importlib-metadata==4.10.1
-    # via
-    #   click
-    #   jsonschema
 isodate==0.6.0
     # via msrest
 jinja2==3.0.3
@@ -290,6 +287,10 @@ ply==3.11
     # via jsonpath-ng
 portalocker==1.7.1
     # via msal-extensions
+prometheus-client==0.13.1
+    # via prometheus-fastapi-instrumentator
+prometheus-fastapi-instrumentator==5.7.1
+    # via -r requirements.in
 protobuf==3.17.3
     # via
     #   google-api-core
@@ -406,16 +407,9 @@ tornado==6.1
 typing-extensions==3.10.0.0
     # via
     #   aiohttp
-    #   aioitertools
-    #   anyio
-    #   asgiref
-    #   importlib-metadata
     #   osdu-data-ecosystem-search
     #   osdu-data-ecosystem-storage
     #   pydantic
-    #   structlog
-    #   uvicorn
-    #   yarl
 urllib3==1.26.6
     # via
     #   botocore
@@ -430,8 +424,6 @@ yarl==1.6.3
     # via aiohttp
 zict==2.0.0
     # via distributed
-zipp==3.7.0
-    # via importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools

requirements_dev.txt

 #
-# This file is autogenerated by pip-compile with python 3.7
+# This file is autogenerated by pip-compile with python 3.8
 # To update, run:
 #
 #    pip-compile requirements_dev.in
@@ -61,14 +61,6 @@ idna==3.2
     #   anyio
     #   requests
     #   rfc3986
-importlib-metadata==4.10.1
-    # via
-    #   -c requirements.txt
-    #   click
-    #   jsonschema
-    #   pep517
-    #   pluggy
-    #   pytest
 iniconfig==1.1.1
     # via pytest
 isodate==0.6.0
@@ -194,22 +186,12 @@ toml==0.10.2
     #   pytest-cov
 tomli==1.2.0
     # via pep517
-typing-extensions==3.10.0.0
-    # via
-    #   -c requirements.txt
-    #   anyio
-    #   importlib-metadata
 urllib3==1.26.6
     # via
     #   -c requirements.txt
     #   requests
 wheel==0.36.2
     # via pip-tools
-zipp==3.7.0
-    # via
-    #   -c requirements.txt
-    #   importlib-metadata
-    #   pep517
 
 # The following packages are considered to be unsafe in a requirements file:
 # pip

tests/unit/app_conf_test.py

@@ -94,20 +94,6 @@ def test_azure_configuration_checker(azure_config_fixture):
     assert "default_data_tenant_credentials" not in variables_dict
 
 
-def test_azure_trace_exporter_created(azure_config_fixture):
-    exporter_name = 'AzureExporter'
-
-    mock_exporter = mock.MagicMock()
-    mock_exporter.configure_mock(**{'exporter_name': exporter_name})
-
-    with mock.patch('app.helper.traces._create_azure_exporter', mock.Mock(return_value=mock_exporter)):
-        exporter = create_exporter('test-service')
-        assert len(exporter.exporters) == 1
-        # ensure called method is azure exporter
-        azure_exporter = exporter.exporters[0]
-        assert azure_exporter.exporter_name == exporter_name
-
-
 def test_gcp_trace_exporter_created(gcp_config_fixture):
     exporter_name = 'StackdriverExporter'