"""Core functionality shared by all JSON loggers"""
### IMPORTS
### ============================================================================
## Future
from __future__ import annotations
## Standard Library
from datetime import datetime, timezone
import importlib
import logging
import re
import sys
from typing import Optional, Union, Callable, List, Dict, Container, Any, Sequence
if sys.version_info >= (3, 10):
from typing import TypeAlias
else:
from typing_extensions import TypeAlias
## Installed
## Application
### CONSTANTS
### ============================================================================
RESERVED_ATTRS: List[str] = [
"args",
"asctime",
"created",
"exc_info",
"exc_text",
"filename",
"funcName",
"levelname",
"levelno",
"lineno",
"module",
"msecs",
"message",
"msg",
"name",
"pathname",
"process",
"processName",
"relativeCreated",
"stack_info",
"thread",
"threadName",
]
"""Default reserved attributes.
These come from the [default attributes of `LogRecord` objects](http://docs.python.org/library/logging.html#logrecord-attributes).
Note:
Although considered a constant, this list is dependent on the Python version due to
different `LogRecord` objects having different attributes in different Python versions.
*Changed in 3.0*: `RESERVED_ATTRS` is now `list[str]` instead of `tuple[str, ...]`.
"""
if sys.version_info >= (3, 12):
# taskName added in python 3.12
RESERVED_ATTRS.append("taskName")
RESERVED_ATTRS.sort()
STYLE_STRING_TEMPLATE_REGEX = re.compile(r"\$\{(.+?)\}", re.IGNORECASE) # $ style
STYLE_STRING_FORMAT_REGEX = re.compile(r"\{(.+?)\}", re.IGNORECASE) # { style
STYLE_PERCENT_REGEX = re.compile(r"%\((.+?)\)", re.IGNORECASE) # % style
## Type Aliases
## -----------------------------------------------------------------------------
OptionalCallableOrStr: TypeAlias = Optional[Union[Callable, str]]
"""Type alias"""
LogRecord: TypeAlias = Dict[str, Any]
"""Type alias"""
### FUNCTIONS
### ============================================================================
def str_to_object(obj: Any) -> Any:
"""Import strings to an object, leaving non-strings as-is.
Args:
obj: the object or string to process
*New in 3.1*
"""
if not isinstance(obj, str):
return obj
module_name, attribute_name = obj.rsplit(".", 1)
return getattr(importlib.import_module(module_name), attribute_name)
def merge_record_extra(
record: logging.LogRecord,
target: Dict,
reserved: Container[str],
rename_fields: Optional[Dict[str, str]] = None,
) -> Dict:
"""
Merges extra attributes from LogRecord object into target dictionary
Args:
record: logging.LogRecord
target: dict to update
reserved: dict or list with reserved keys to skip
rename_fields: an optional dict, used to rename field names in the output.
e.g. Rename `levelname` to `log.level`: `{'levelname': 'log.level'}`
*Changed in 3.1*: `reserved` is now `Container[str]`.
"""
if rename_fields is None:
rename_fields = {}
for key, value in record.__dict__.items():
# this allows to have numeric keys
if key not in reserved and not (hasattr(key, "startswith") and key.startswith("_")):
target[rename_fields.get(key, key)] = value
return target
### CLASSES
### ============================================================================
class BaseJsonFormatter(logging.Formatter):
"""Base class for all formatters
Must not be used directly.
*New in 3.1*
*Changed in 3.2*: `defaults` argument is no longer ignored.
*Added in UNRELEASED*: `exc_info_as_array` and `stack_info_as_array` options are added.
"""
_style: Union[logging.PercentStyle, str] # type: ignore[assignment]
## Parent Methods
## -------------------------------------------------------------------------
# pylint: disable=too-many-arguments,super-init-not-called
def __init__(
self,
fmt: Optional[str] = None,
datefmt: Optional[str] = None,
style: str = "%",
validate: bool = True,
*,
prefix: str = "",
rename_fields: Optional[Dict[str, str]] = None,
rename_fields_keep_missing: bool = False,
static_fields: Optional[Dict[str, Any]] = None,
reserved_attrs: Optional[Sequence[str]] = None,
timestamp: Union[bool, str] = False,
defaults: Optional[Dict[str, Any]] = None,
exc_info_as_array: bool = False,
stack_info_as_array: bool = False,
) -> None:
"""
Args:
fmt: string representing fields to log
datefmt: format to use when formatting `asctime` field
style: how to extract log fields from `fmt`
validate: validate `fmt` against style, if implementing a custom `style` you
must set this to `False`.
defaults: a dictionary containing default fields that are added before all other fields and
may be overridden. The supplied fields are still subject to `rename_fields`.
prefix: an optional string prefix added at the beginning of
the formatted string
rename_fields: an optional dict, used to rename field names in the output.
Rename `message` to `@message`: `{'message': '@message'}`
rename_fields_keep_missing: When renaming fields, include missing fields in the output.
static_fields: an optional dict, used to add fields with static values to all logs
reserved_attrs: an optional list of fields that will be skipped when
outputting json log record. Defaults to [all log record attributes][pythonjsonlogger.core.RESERVED_ATTRS].
timestamp: an optional string/boolean field to add a timestamp when
outputting the json log record. If string is passed, timestamp will be added
to log record using string as key. If True boolean is passed, timestamp key
will be "timestamp". Defaults to False/off.
exc_info_as_array: break the exc_info into a list of lines based on line breaks.
stack_info_as_array: break the stack_info into a list of lines based on line breaks.
*Changed in 3.1*:
- you can now use custom values for style by setting validate to `False`.
The value is stored in `self._style` as a string. The `parse` method will need to be
overridden in order to support the new style.
- Renaming fields now preserves the order that fields were added in and avoids adding
missing fields. The original behaviour, missing fields have a value of `None`, is still
available by setting `rename_fields_keep_missing` to `True`.
"""
## logging.Formatter compatibility
## ---------------------------------------------------------------------
# Note: validate added in 3.8, defaults added in 3.10
if style in logging._STYLES:
_style = logging._STYLES[style][0](fmt) # type: ignore[operator]
if validate:
_style.validate()
self._style = _style
self._fmt = _style._fmt
elif not validate:
self._style = style
self._fmt = fmt
else:
raise ValueError(f"Style must be one of: {','.join(logging._STYLES.keys())}")
self.datefmt = datefmt
## JSON Logging specific
## ---------------------------------------------------------------------
self.prefix = prefix
self.rename_fields = rename_fields if rename_fields is not None else {}
self.rename_fields_keep_missing = rename_fields_keep_missing
self.static_fields = static_fields if static_fields is not None else {}
self.reserved_attrs = set(reserved_attrs if reserved_attrs is not None else RESERVED_ATTRS)
self.timestamp = timestamp
self._required_fields = self.parse()
self._skip_fields = set(self._required_fields)
self._skip_fields.update(self.reserved_attrs)
self.defaults = defaults if defaults is not None else {}
self.exc_info_as_array = exc_info_as_array
self.stack_info_as_array = stack_info_as_array
return
def format(self, record: logging.LogRecord) -> str:
"""Formats a log record and serializes to json
Args:
record: the record to format
"""
message_dict: Dict[str, Any] = {}
# TODO: logging.LogRecord.msg and logging.LogRecord.message in typeshed
# are always type of str. We shouldn't need to override that.
if isinstance(record.msg, dict):
message_dict = record.msg
record.message = ""
else:
record.message = record.getMessage()
# only format time if needed
if "asctime" in self._required_fields:
record.asctime = self.formatTime(record, self.datefmt)
# Display formatted exception, but allow overriding it in the
# user-supplied dict.
if record.exc_info and not message_dict.get("exc_info"):
message_dict["exc_info"] = self.formatException(record.exc_info)
if not message_dict.get("exc_info") and record.exc_text:
message_dict["exc_info"] = record.exc_text
# Display formatted record of stack frames
# default format is a string returned from :func:`traceback.print_stack`
if record.stack_info and not message_dict.get("stack_info"):
message_dict["stack_info"] = self.formatStack(record.stack_info)
log_record: LogRecord = {}
self.add_fields(log_record, record, message_dict)
log_record = self.process_log_record(log_record)
return self.serialize_log_record(log_record)
## JSON Formatter Specific Methods
## -------------------------------------------------------------------------
def parse(self) -> List[str]:
"""Parses format string looking for substitutions
This method is responsible for returning a list of fields (as strings)
to include in all log messages.
You can support custom styles by overriding this method.
Returns:
list of fields to be extracted and serialized
"""
if isinstance(self._style, logging.StringTemplateStyle):
formatter_style_pattern = STYLE_STRING_TEMPLATE_REGEX
elif isinstance(self._style, logging.StrFormatStyle):
formatter_style_pattern = STYLE_STRING_FORMAT_REGEX
elif isinstance(self._style, logging.PercentStyle):
# PercentStyle is parent class of StringTemplateStyle and StrFormatStyle
# so it must be checked last.
formatter_style_pattern = STYLE_PERCENT_REGEX
else:
raise ValueError(f"Style {self._style!r} is not supported")
if self._fmt:
return formatter_style_pattern.findall(self._fmt)
return []
def serialize_log_record(self, log_record: LogRecord) -> str:
"""Returns the final representation of the log record.
Args:
log_record: the log record
"""
return self.prefix + self.jsonify_log_record(log_record)
def add_fields(
self,
log_record: Dict[str, Any],
record: logging.LogRecord,
message_dict: Dict[str, Any],
) -> None:
"""Extract fields from a LogRecord for logging
This method can be overridden to implement custom logic for adding fields.
Args:
log_record: data that will be logged
record: the record to extract data from
message_dict: dictionary that was logged instead of a message. e.g
`logger.info({"is_this_message_dict": True})`
"""
for field in self.defaults:
log_record[self._get_rename(field)] = self.defaults[field]
for field in self._required_fields:
log_record[self._get_rename(field)] = record.__dict__.get(field)
for data_dict in [self.static_fields, message_dict]:
for key, value in data_dict.items():
log_record[self._get_rename(key)] = value
merge_record_extra(
record,
log_record,
reserved=self._skip_fields,
rename_fields=self.rename_fields,
)
if self.timestamp:
key = self.timestamp if isinstance(self.timestamp, str) else "timestamp"
log_record[self._get_rename(key)] = datetime.fromtimestamp(
record.created, tz=timezone.utc
)
if self.rename_fields_keep_missing:
for field in self.rename_fields.values():
if field not in log_record:
log_record[field] = None
return
def _get_rename(self, key: str) -> str:
return self.rename_fields.get(key, key)
# Child Methods
# ..........................................................................
def jsonify_log_record(self, log_record: LogRecord) -> str:
"""Convert this log record into a JSON string.
Child classes MUST override this method.
Args:
log_record: the data to serialize
"""
raise NotImplementedError()
def process_log_record(self, log_record: LogRecord) -> LogRecord:
"""Custom processing of the log record.
Child classes can override this method to alter the log record before it
is serialized.
Args:
log_record: incoming data
"""
return log_record
def formatException(self, ei) -> Union[str, list[str]]: # type: ignore
"""Format and return the specified exception information.
If exc_info_as_array is set to True, This method returns an array of strings.
"""
exception_info_str = super().formatException(ei)
return exception_info_str.splitlines() if self.exc_info_as_array else exception_info_str
def formatStack(self, stack_info) -> Union[str, list[str]]: # type: ignore
"""Format and return the specified stack information.
If stack_info_as_array is set to True, This method returns an array of strings.
"""
stack_info_str = super().formatStack(stack_info)
return stack_info_str.splitlines() if self.stack_info_as_array else stack_info_str
