logger.py 6.61 KB
Newer Older
ethiraj krishnamanaidu's avatar
ethiraj krishnamanaidu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2021 Schlumberger
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
import logging
import traceback
import sys
18
import rapidjson
Luc Yriarte's avatar
Luc Yriarte committed
19

20
import structlog
21
from structlog.contextvars import merge_contextvars
Luc Yriarte's avatar
Luc Yriarte committed
22
from opencensus.trace import config_integration
23
24
25
26

from app.conf import Config
from app.utils import get_or_create_ctx
from app.helper.utils import rename_cloud_role_func
Luc Yriarte's avatar
Luc Yriarte committed
27
28
29
30
31
32

_LOGGER = None


def get_logger():
    return _LOGGER
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60


class StackDriverRenderer(object):
    def __init__(self, service_name=None):
        self.service_name = service_name

    def __call__(self, _, method, event_dict):
        if self.service_name:
            event_dict['serviceContext'] = {'service': self.service_name}

        # rename event to msg
        if 'event' in event_dict:
            event_dict['message'] = event_dict['event']
            del event_dict['event']

        # Required by stackdriver to display level of error accordingly
        event_dict.setdefault("severity", method)

        if method == 'error' or method == 'critical':
            # Enable display of this error in 'Error reporting' in GCP
            event_dict['@type'] = 'type.googleapis.com/google.devtools.clouderrorreporting.v1beta1.ReportedErrorEvent'
            if sys.exc_info()[0]:
                # check if an exception exist https://docs.python.org/2/library/sys.html#sys.exc_info
                event_dict['stack_trace'] = traceback.format_exc()

        return event_dict


Luc Yriarte's avatar
Luc Yriarte committed
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
class AzureContextLoggerAdapter(logging.LoggerAdapter):
    """
    This adapter adds contextual information into messages to be logged in Azure monitoring.
    It aims to add as custom properties contextual fields, following this instructions:
    https://docs.microsoft.com/en-us/azure/azure-monitor/app/opencensus-python
    """

    @staticmethod
    def _set_extra_attrs(properties):
        ctx = get_or_create_ctx()

        properties.setdefault('correlation-id', ctx.correlation_id)
        properties.setdefault('request-id', ctx.request_id)
        properties.setdefault('data-partition-id', ctx.partition_id)
        properties.setdefault('app-key', ctx.app_key)
        properties.setdefault('api-key', ctx.api_key)

    def process(self, msg, kwargs):
        """
        Retrieve context created in basic middleware from request info to append them
        in log message as custom attributes
        """
        custom_properties = dict()
        self._set_extra_attrs(custom_properties)
        kwargs['extra'] = dict(custom_dimensions=custom_properties)

        return msg, kwargs

89

90
def init_logger(service_name):
Luc Yriarte's avatar
Luc Yriarte committed
91
92
93
    global _LOGGER

    if Config.cloud_provider.value == 'az':
94
        _LOGGER = create_azure_logger(service_name)
Luc Yriarte's avatar
Luc Yriarte committed
95
    elif Config.cloud_provider.value == 'gcp':
96
        _LOGGER = create_gcp_logger(service_name)
Luc Yriarte's avatar
Luc Yriarte committed
97
98
99
100
101
102
103
    else:
        logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
        _LOGGER = logging.getLogger(__name__)

    return _LOGGER


104
def create_azure_logger(service_name):
105
    from opencensus.ext.azure.log_exporter import AzureLogHandler
Luc Yriarte's avatar
Luc Yriarte committed
106
107
108
109
110
111
112
    """
    Create logger with two handlers:
     - AzureLogHandler: to see Dependencies, Requests, Traces and Exception into Azure monitoring
     - [default] StreamHandler (c.f. logging.basicConfig() ) to see all logs into the std.out captured in container logs

     returns logger configured wrapped into ContextLoggerAdapter
    """
Yannick's avatar
Yannick committed
113
    from opencensus.ext.azure.log_exporter import AzureLogHandler
Luc Yriarte's avatar
Luc Yriarte committed
114
115
    config_integration.trace_integrations(['logging'])

116
117
    # stdout handler for direct logging output to stdout.
    stdout_handler = logging.StreamHandler(sys.stdout)
Luc Yriarte's avatar
Luc Yriarte committed
118

119
    #  AzurelogHandler for logging to azure appinsight
Luc Yriarte's avatar
Luc Yriarte committed
120
    key = Config.get('az_ai_instrumentation_key')
121

Vaibhavi Kamani's avatar
Vaibhavi Kamani committed
122
    if key:
123
124
125
126
127
        logger_level = Config.get('az_logger_level')
        az_handler = AzureLogHandler(connection_string=f'InstrumentationKey={key}')
        az_handler.setLevel(logging.getLevelName(logger_level))
        az_handler.add_telemetry_processor(rename_cloud_role_func(service_name))
        
128
    # Acquire the logger for azure library
Vaibhavi Kamani's avatar
Vaibhavi Kamani committed
129
    __getLogger(logger_name='azure', log_level=logging.INFO, stdout_handler=stdout_handler)
130
131

    # Acquire the logger for osdu-core-lib-python-azure
Vaibhavi Kamani's avatar
Vaibhavi Kamani committed
132
133
    __getLogger(logger_name='osdu_az', log_level=logging.INFO, stdout_handler=stdout_handler, 
                az_handler=az_handler)
134
135

    # Acquire the logger for wdms
Vaibhavi Kamani's avatar
Vaibhavi Kamani committed
136
137
    __getLogger(logger_name=__name__, log_level=logging.INFO, stdout_handler=stdout_handler, 
                az_handler=az_handler)
Luc Yriarte's avatar
Luc Yriarte committed
138

139
    # return wdms logger with Context adapter
Luc Yriarte's avatar
Luc Yriarte committed
140
141
142
    return AzureContextLoggerAdapter(logger, extra=dict())


143
def create_gcp_logger(service_name):
Luc Yriarte's avatar
Luc Yriarte committed
144
145
146
147
148
    """
    Initialize structlog with following configuration:
        - Make logs compatible with Stackdriver
        - if dev_mode, display stacktrace out of json item
    Returns structlog
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
    """

    structlog.configure(
        processors=[
            StackDriverRenderer(service_name=service_name),
            merge_contextvars,
            structlog.stdlib.filter_by_level,
            structlog.stdlib.add_logger_name,
            structlog.stdlib.PositionalArgumentsFormatter(),
            structlog.processors.TimeStamper(fmt="iso"),
            structlog.processors.StackInfoRenderer(),
            structlog.processors.format_exc_info,
            structlog.processors.UnicodeDecoder(),
            structlog.processors.JSONRenderer(serializer=rapidjson.dumps)
        ],
        context_class=dict,
        logger_factory=structlog.stdlib.LoggerFactory(),
        wrapper_class=structlog.stdlib.BoundLogger,
        cache_logger_on_first_use=True,
    )

Luc Yriarte's avatar
Luc Yriarte committed
170
    my_logger = structlog.getLogger(__name__)
171
172
173

    ch = logging.StreamHandler(sys.stdout)
    ch.setLevel(logging.DEBUG)
Luc Yriarte's avatar
Luc Yriarte committed
174
    ch.setFormatter(logging.Formatter('%(message)s'))
175
176
    my_logger.addHandler(ch)

Luc Yriarte's avatar
Luc Yriarte committed
177
178
    std_ddms_app = logging.getLogger(__name__)
    # avoid double logging by the root logger
179
180
181
    std_ddms_app.propagate = False

    return my_logger
Vaibhavi Kamani's avatar
Vaibhavi Kamani committed
182
183
184
185
186
187
188
189

def __getLogger(logger_name, log_level, stdout_handler, az_handler=None):
    logger = logging.getLogger(logger_name)
    logger.setLevel(log_level)
    logger.addHandler(stdout_handler)

    if az_handler:
        logger.addHandler(az_handler)