diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..04f48a4 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @RomanR-dev diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..75da162 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,41 @@ +name: Python Tests with Tox + +on: + pull_request: + branches: [ master ] + + +jobs: + test: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.11'] + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Install tox + run: | + python -m pip install --upgrade pip + pip install tox tox-gh-actions + + - name: Run tox + run: tox + + - name: Code Coverage Summary Report + uses: irongut/CodeCoverageSummary@v1.3.0 + with: + filename: coverage.xml + badge: true + format: markdown + thresholds: '80 100' + output: 'both' + fail_below_min: 'true' + indicators: 'true' diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dadfade..0000000 --- a/.travis.yml +++ /dev/null @@ -1,8 +0,0 @@ -language: python -python: - - "3.6" - - "3.7" - - "3.8" -dist: xenial -install: pip install tox tox-venv tox-travis -script: tox \ No newline at end of file diff --git a/README.md b/README.md index 54c7642..b54f8c1 100644 --- a/README.md +++ b/README.md @@ -1,90 +1,122 @@ -python-logging-loki -=================== +# 🚀 python-logging-loki-v2 -[![PyPI version](https://img.shields.io/pypi/v/python-logging-loki.svg)](https://pypi.org/project/python-logging-loki/) -[![Python version](https://img.shields.io/badge/python-3.6%20%7C%203.7%20%7C%203.8-blue.svg)](https://www.python.org/) -[![License](https://img.shields.io/pypi/l/python-logging-loki.svg)](https://opensource.org/licenses/MIT) -[![Build Status](https://travis-ci.org/GreyZmeem/python-logging-loki.svg?branch=master)](https://travis-ci.org/GreyZmeem/python-logging-loki) +> Modern Python logging handler for Grafana Loki -Python logging handler for Loki. -https://grafana.com/loki +[![PyPI version](https://img.shields.io/pypi/v/python-logging-loki-v2.svg)](https://pypi.org/project/python-logging-loki-v2/) +[![Python](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/) + +## Documented by Grafana: https://github.com/grafana/loki/pull/16397 + +Send Python logs directly to [Grafana Loki](https://grafana.com/loki) with minimal configuration. + +--- + +## ✨ Features + +- 📤 **Direct Integration** - Send logs straight to Loki +- 🔐 **Authentication Support** - Basic auth and custom headers +- 🏷️ **Custom Labels** - Flexible tagging system +- ⚡ **Async Support** - Non-blocking queue handler included +- 🔒 **SSL Verification** - Configurable SSL/TLS settings +- 🎯 **Multi-tenant** - Support for Loki multi-tenancy + +--- + +## 📦 Installation -Installation -============ ```bash -pip install python-logging-loki +pip install python-logging-loki-v2 ``` -Usage -===== +--- + +## 🎯 Quick Start + +### Basic Usage ```python import logging import logging_loki - handler = logging_loki.LokiHandler( - url="https://my-loki-instance/loki/api/v1/push", - tags={"application": "my-app"}, + url="https://loki.example.com/loki/api/v1/push", + tags={"app": "my-application"}, auth=("username", "password"), - version="1", + version="2" ) -logger = logging.getLogger("my-logger") +logger = logging.getLogger("my-app") logger.addHandler(handler) -logger.error( - "Something happened", - extra={"tags": {"service": "my-service"}}, -) +logger.info("Application started", extra={"tags": {"env": "production"}}) ``` -Example above will send `Something happened` message along with these labels: -- Default labels from handler -- Message level as `serverity` -- Logger's name as `logger` -- Labels from `tags` item of `extra` dict +### Async/Non-blocking Mode -The given example is blocking (i.e. each call will wait for the message to be sent). -But you can use the built-in `QueueHandler` and` QueueListener` to send messages in a separate thread. +For high-throughput applications, use the queue handler to avoid blocking: ```python import logging.handlers import logging_loki from multiprocessing import Queue - -queue = Queue(-1) -handler = logging.handlers.QueueHandler(queue) -handler_loki = logging_loki.LokiHandler( - url="https://my-loki-instance/loki/api/v1/push", - tags={"application": "my-app"}, - auth=("username", "password"), - version="1", +handler = logging_loki.LokiQueueHandler( + Queue(-1), + url="https://loki.example.com/loki/api/v1/push", + tags={"app": "my-application"}, + version="2" ) -logging.handlers.QueueListener(queue, handler_loki) -logger = logging.getLogger("my-logger") +logger = logging.getLogger("my-app") logger.addHandler(handler) -logger.error(...) +logger.info("Non-blocking log message") ``` -Or you can use `LokiQueueHandler` shortcut, which will automatically create listener and handler. +--- -```python -import logging.handlers -import logging_loki -from multiprocessing import Queue +## ⚙️ Configuration Options +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `url` | `str` | *required* | Loki push endpoint URL | +| `tags` | `dict` | `{}` | Default labels for all logs | +| `auth` | `tuple` | `None` | Basic auth credentials `(username, password)` | +| `headers` | `dict` | `None` | Custom HTTP headers (e.g., for multi-tenancy) | +| `version` | `str` | `"1"` | Loki API version (`"0"`, `"1"`, or `"2"`) | +| `verify_ssl` | `bool` | `True` | Enable/disable SSL certificate verification | -handler = logging_loki.LokiQueueHandler( - Queue(-1), - url="https://my-loki-instance/loki/api/v1/push", - tags={"application": "my-app"}, - auth=("username", "password"), - version="1", +--- + +## 🏷️ Labels + +Logs are automatically labeled with: +- **severity** - Log level (INFO, ERROR, etc.) +- **logger** - Logger name +- **Custom tags** - From handler and `extra={"tags": {...}}` + +```python +logger.error( + "Database connection failed", + extra={"tags": {"service": "api", "region": "us-east"}} ) +``` -logger = logging.getLogger("my-logger") -logger.addHandler(handler) -logger.error(...) +--- + +## 🔐 Multi-tenant Setup + +```python +handler = logging_loki.LokiHandler( + url="https://loki.example.com/loki/api/v1/push", + headers={"X-Scope-OrgID": "tenant-1"}, + tags={"app": "my-app"} +) ``` + +--- +Based on [python-logging-loki](https://github.com/GreyZmeem/python-logging-loki) by GreyZmeem. + +### Contributing + +Contributions are welcome! Please feel free to submit a Pull Request. + +--- diff --git a/logging_loki/__init__.py b/logging_loki/__init__.py index f9d6949..2e7fef4 100644 --- a/logging_loki/__init__.py +++ b/logging_loki/__init__.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -from logging_loki.handlers import LokiHandler -from logging_loki.handlers import LokiQueueHandler +from logging_loki.handlers import LokiHandler, LokiQueueHandler __all__ = ["LokiHandler", "LokiQueueHandler"] __version__ = "0.3.1" diff --git a/logging_loki/emitter.py b/logging_loki/emitter.py index 949ceea..aeacea6 100644 --- a/logging_loki/emitter.py +++ b/logging_loki/emitter.py @@ -6,11 +6,7 @@ import logging import time from logging.config import ConvertingDict -from typing import Any -from typing import Dict -from typing import List -from typing import Optional -from typing import Tuple +from typing import Any, Dict, List, Optional, Tuple import requests import rfc3339 @@ -30,7 +26,7 @@ class LokiEmitter(abc.ABC): label_replace_with = const.label_replace_with session_class = requests.Session - def __init__(self, url: str, tags: Optional[dict] = None, auth: BasicAuth = None): + def __init__(self, url: str, tags: dict | None = None, auth: BasicAuth = None, headers: dict | None = None, verify_ssl: bool = True): """ Create new Loki emitter. @@ -38,7 +34,7 @@ def __init__(self, url: str, tags: Optional[dict] = None, auth: BasicAuth = None url: Endpoint used to send log entries to Loki (e.g. `https://my-loki-instance/loki/api/v1/push`). tags: Default tags added to every log record. auth: Optional tuple with username and password for basic HTTP authentication. - + headers: Optional dict with HTTP headers to send. """ #: Tags that will be added to all records handled by this handler. self.tags = tags or {} @@ -46,13 +42,17 @@ def __init__(self, url: str, tags: Optional[dict] = None, auth: BasicAuth = None self.url = url #: Optional tuple with username and password for basic authentication. self.auth = auth + #: Optional headers for post request + self.headers = headers or {} + #: Verify the host's ssl certificate + self.verify_ssl = verify_ssl - self._session: Optional[requests.Session] = None + self._session: requests.Session | None = None def __call__(self, record: logging.LogRecord, line: str): """Send log record to Loki.""" payload = self.build_payload(record, line) - resp = self.session.post(self.url, json=payload) + resp = self.session.post(self.url, json=payload, headers=self.headers, verify=self.verify_ssl) if resp.status_code != self.success_response_code: raise ValueError("Unexpected Loki API response status code: {0}".format(resp.status_code)) @@ -135,9 +135,26 @@ def build_payload(self, record: logging.LogRecord, line) -> dict: """Build JSON payload with a log entry.""" labels = self.build_tags(record) ns = 1e9 - ts = str(int(time.time() * ns)) + ts = str(int(record.created * ns)) stream = { "stream": labels, "values": [[ts, line]], } return {"streams": [stream]} + + +class LokiEmitterV2(LokiEmitterV1): + """ + Emitter for Loki >= 0.4.0. + Enables passing additional headers to requests + """ + + def __init__(self, url: str, tags: dict | None = None, auth: BasicAuth = None, headers: dict = None, verify_ssl: bool = True): + super().__init__(url, tags, auth, headers, verify_ssl) + + def __call__(self, record: logging.LogRecord, line: str): + """Send log record to Loki.""" + payload = self.build_payload(record, line) + resp = self.session.post(self.url, json=payload, headers=self.headers, verify=self.verify_ssl) + if resp.status_code != self.success_response_code: + raise ValueError("Unexpected Loki API response status code: {0}".format(resp.status_code)) diff --git a/logging_loki/handlers.py b/logging_loki/handlers.py index 74a55cb..030aa2b 100644 --- a/logging_loki/handlers.py +++ b/logging_loki/handlers.py @@ -2,15 +2,11 @@ import logging import warnings -from logging.handlers import QueueHandler -from logging.handlers import QueueListener +from logging.handlers import QueueHandler, QueueListener from queue import Queue -from typing import Dict -from typing import Optional -from typing import Type +from typing import Dict, Type -from logging_loki import const -from logging_loki import emitter +from logging_loki import const, emitter class LokiQueueHandler(QueueHandler): @@ -19,7 +15,7 @@ class LokiQueueHandler(QueueHandler): def __init__(self, queue: Queue, **kwargs): """Create new logger handler with the specified queue and kwargs for the `LokiHandler`.""" super().__init__(queue) - self.handler = LokiHandler(**kwargs) # noqa: WPS110 + self.handler = LokiHandler(**kwargs) self.listener = QueueListener(self.queue, self.handler) self.listener.start() @@ -31,18 +27,9 @@ class LokiHandler(logging.Handler): `Loki API `_ """ - emitters: Dict[str, Type[emitter.LokiEmitter]] = { - "0": emitter.LokiEmitterV0, - "1": emitter.LokiEmitterV1, - } - - def __init__( - self, - url: str, - tags: Optional[dict] = None, - auth: Optional[emitter.BasicAuth] = None, - version: Optional[str] = None, - ): + emitters: Dict[str, Type[emitter.LokiEmitter]] = {"0": emitter.LokiEmitterV0, "1": emitter.LokiEmitterV1, "2": emitter.LokiEmitterV2} + + def __init__(self, url: str, tags: dict | None = None, auth: emitter.BasicAuth | None = None, version: str | None = None, headers: dict | None = None, verify_ssl: bool = True): """ Create new Loki logging handler. @@ -51,6 +38,7 @@ def __init__( tags: Default tags added to every log record. auth: Optional tuple with username and password for basic HTTP authentication. version: Version of Loki emitter to use. + verify_ssl: If set to False, the endpoint's SSL certificates are not verified """ super().__init__() @@ -67,7 +55,7 @@ def __init__( version = version or const.emitter_ver if version not in self.emitters: raise ValueError("Unknown emitter version: {0}".format(version)) - self.emitter = self.emitters[version](url, tags, auth) + self.emitter = self.emitters[version](url, tags, auth, headers, verify_ssl) def handleError(self, record): # noqa: N802 """Close emitter and let default handler take actions on error.""" diff --git a/pyproject.toml b/pyproject.toml index 5428332..c05ed4d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,39 @@ [build-system] requires = ["setuptools >= 40.0.0"] build-backend = "setuptools.build_meta" + + +[tool.ruff] +line-length = 200 + +[tool.ruff.lint] +# Enable flake8-style rules and more +# You can customize this based on wemake-python-styleguide rules you want +select = [ + "E", # pycodestyle errors + "W", # pycodestyle warnings + "F", # pyflakes + "I", # isort + "N", # pep8-naming + "UP", # pyupgrade + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "SIM", # flake8-simplify +] + +# Ignore specific rules if needed +ignore = [ + "UP009", # UTF-8 encoding declaration is unnecessary + "UP035", # typing.X is deprecated, use x instead + "UP006", # Use x instead of X for type annotation + "UP030", # Use implicit references for positional format fields + "UP032", # Use f-string instead of format call + "UP015", # Unnecessary mode argument + "UP045", # New union syntax + "B019", # Use of functools.lru_cache on methods can lead to memory leaks + "B028", # No explicit stacklevel keyword argument found +] + +[tool.ruff.format] +# Use double quotes (ruff default, similar to black) +quote-style = "double" diff --git a/setup.py b/setup.py index 153a2b5..2aebd6d 100644 --- a/setup.py +++ b/setup.py @@ -2,32 +2,29 @@ import setuptools -with open("README.md", "r") as fh: +with open("README.md", encoding="utf-8") as fh: long_description = fh.read() setuptools.setup( - name="python-logging-loki", - version="0.3.1", - description="Python logging handler for Grafana Loki.", + name="python-logging-loki-v2", + version="1.1.1", + description="Python logging handler for Grafana Loki", long_description=long_description, long_description_content_type="text/markdown", license="MIT", - author="Andrey Maslov", - author_email="greyzmeem@gmail.com", - url="https://github.com/greyzmeem/python-logging-loki", + author="Roman Rapoport", + author_email="cryos10@gmail.com", + url="https://github.com/RomanR-dev/python-logging-loki", packages=setuptools.find_packages(exclude=("tests",)), - python_requires=">=3.6", + python_requires=">=3.11", install_requires=["rfc3339>=6.1", "requests"], classifiers=[ - "Development Status :: 4 - Beta", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: System :: Logging", "Topic :: Internet :: WWW/HTTP", diff --git a/tests/test_emitter_v0.py b/tests/test_emitter_v0.py index 0aafd8d..dff2e2d 100644 --- a/tests/test_emitter_v0.py +++ b/tests/test_emitter_v0.py @@ -157,7 +157,7 @@ def test_session_is_closed(emitter_v0): emitter(create_record(), "") emitter.close() session().close.assert_called_once() - assert emitter._session is None # noqa: WPS437 + assert emitter._session is None def test_can_build_tags_from_converting_dict(emitter_v0): diff --git a/tests/test_emitter_v1.py b/tests/test_emitter_v1.py index b5656e1..2963474 100644 --- a/tests/test_emitter_v1.py +++ b/tests/test_emitter_v1.py @@ -152,7 +152,7 @@ def test_session_is_closed(emitter_v1): emitter(create_record(), "") emitter.close() session().close.assert_called_once() - assert emitter._session is None # noqa: WPS437 + assert emitter._session is None def test_can_build_tags_from_converting_dict(emitter_v1): diff --git a/tests/test_emitter_v2.py b/tests/test_emitter_v2.py new file mode 100644 index 0000000..88049b8 --- /dev/null +++ b/tests/test_emitter_v2.py @@ -0,0 +1,123 @@ +# -*- coding: utf-8 -*- + +import logging +from typing import Tuple +from unittest.mock import MagicMock + +import pytest + +from logging_loki.emitter import LokiEmitterV2 + +emitter_url: str = "https://example.net/loki/api/v1/push/" +headers = {"X-Scope-OrgID": "some_tenant"} +record_kwargs = { + "name": "test", + "level": logging.WARNING, + "fn": "", + "lno": "", + "msg": "Test", + "args": None, + "exc_info": None, +} + + +@pytest.fixture() +def emitter_v2() -> Tuple[LokiEmitterV2, MagicMock]: + """Create v2 emitter with mocked http session.""" + response = MagicMock() + response.status_code = LokiEmitterV2.success_response_code + session = MagicMock() + session().post = MagicMock(return_value=response) + + instance = LokiEmitterV2(url=emitter_url, headers=headers) + instance.session_class = session + + return instance, session + + +def test_init(): + LokiEmitterV2(url=emitter_url, headers=headers) + LokiEmitterV2(url=emitter_url, headers=headers, tags={}) + LokiEmitterV2(url=emitter_url, headers=headers, tags={}, verify_ssl=True) + + +@pytest.fixture() +def emitter_v2_no_headers() -> Tuple[LokiEmitterV2, MagicMock]: + """Create v2 emitter with mocked http session.""" + response = MagicMock() + response.status_code = LokiEmitterV2.success_response_code + session = MagicMock() + session().post = MagicMock(return_value=response) + + instance = LokiEmitterV2(url=emitter_url) + instance.session_class = session + + return instance, session + + +def create_record(**kwargs) -> logging.LogRecord: + """Create test logging record.""" + log = logging.Logger(__name__) + return log.makeRecord(**{**record_kwargs, **kwargs}) + + +def get_stream(session: MagicMock) -> dict: + """Return first stream item from json payload.""" + kwargs = session().post.call_args[1] + streams = kwargs["json"]["streams"] + return streams[0] + + +def get_request(session: MagicMock) -> dict: + kwargs = session().post.call_args[1] + return kwargs + + +def test_record_sent_to_emitter_url(emitter_v2): + emitter, session = emitter_v2 + emitter(create_record(), "") + + got = session().post.call_args + assert got[0][0] == emitter_url + + +def test_default_tags_added_to_payload(emitter_v2): + emitter, session = emitter_v2 + emitter.tags = {"app": "emitter"} + emitter(create_record(), "") + + stream = get_stream(session) + level = logging.getLevelName(record_kwargs["level"]).lower() + expected = { + emitter.level_tag: level, + emitter.logger_tag: record_kwargs["name"], + "app": "emitter", + } + assert stream["stream"] == expected + + +def test_headers_added(emitter_v2): + emitter, session = emitter_v2 + emitter.tags = {"app": "emitter"} + emitter(create_record(), "") + + kwargs = get_request(session) + assert kwargs["headers"]["X-Scope-OrgID"] == headers["X-Scope-OrgID"] + + +def test_no_headers_added(emitter_v2_no_headers): + emitter, session = emitter_v2_no_headers + emitter.tags = {"app": "emitter"} + emitter(create_record(), "") + + kwargs = get_request(session) + assert kwargs["headers"] is not None and kwargs["headers"] == {} + + +def test_soemthing_fun(): + import os + + a = "a" + b = "b" + c = "/c" + print(os.path.join(a, b, c)) diff --git a/tox.ini b/tox.ini index a611104..658ecc4 100644 --- a/tox.ini +++ b/tox.ini @@ -1,16 +1,8 @@ [tox] -envlist = - py{36,37,38}, - flake8, - black +# List of Python versions to test against +envlist = py311,py312,ruff isolated_build = true -[travis] -python = - 3.6: py36 - 3.7: py37, flake8, black - 3.8: py38 - [testenv] setenv = LC_ALL = en_US.UTF-8 @@ -19,16 +11,20 @@ deps = pytest coverage freezegun -commands = coverage run -m pytest [] - -[testenv:flake8] -skip_install = true -basepython = python3.7 -deps = wemake-python-styleguide -commands = flake8 . +commands = + coverage run -m pytest [] + coverage report + coverage xml -[testenv:black] +[testenv:ruff] skip_install = true -basepython = python3.7 -deps = black==19.10b0 -commands = black --check --diff -l 120 -t py36 . +# Use whatever python3 is available on your system +basepython = python3.11 +deps = ruff +commands = + # First fix auto-fixable issues + ruff check --fix . + ruff format . + # Then check for any remaining issues + ruff check . + ruff format --check .