diff --git a/.env.benefit-backend.example b/.env.benefit-backend.example index a1b1258d14..fedf76568b 100644 --- a/.env.benefit-backend.example +++ b/.env.benefit-backend.example @@ -88,16 +88,15 @@ DEFAULT_FROM_EMAIL='Helsinki-lisä ' # Sentry configuration for scrubbing sensitive data # Values set here are for local development, no data is scrubbed. -ENTRY_ATTACH_STACKTRACE=True +SENTRY_ATTACH_STACKTRACE=True SENTRY_MAX_BREADCRUMBS=100 SENTRY_REQUEST_BODIES=always SENTRY_SEND_DEFAULT_PII=True SENTRY_WITH_LOCALS=True - -# For connecting to Sentry, get the correct DSN from Sentry or from a team member SENTRY_DSN= -# The environment that the application is shown under in Sentry -# local / development / testing +SENTRY_PROFILE_SESSION_SAMPLE_RATE=0 +SENTRY_TRACES_SAMPLE_RATE=0 +SENTRY_TRACES_IGNORE_PATHS=/healthz,/readiness SENTRY_ENVIRONMENT=local # for Mailpit inbox @@ -112,8 +111,8 @@ AHJO_REDIRECT_URL= DISABLE_AHJO_SAFE_LIST_CHECK=True AHJO_TEST_USER_FIRST_NAME= -AHJO_TEST_USER_LAST_NAME= -AHJO_TEST_USER_AD_USERNAME= +AHJO_TEST_USER_LAST_NAME= +AHJO_TEST_USER_AD_USERNAME= ENABLE_CLAMAV=1 CLAMAV_URL=http://localhost:8080/api/v1 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 44df1fcb0e..8095155740 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - id: check-toml - id: check-added-large-files - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.13.1 + rev: v0.14.6 hooks: # Run the linter. - id: ruff-check @@ -21,13 +21,13 @@ repos: # Run the formatter. - id: ruff-format - repo: https://github.com/alessandrojcm/commitlint-pre-commit-hook - rev: v9.22.0 + rev: v9.23.0 hooks: - id: commitlint stages: [commit-msg, manual] additional_dependencies: ["@commitlint/config-conventional"] - repo: https://github.com/koalaman/shellcheck-precommit - rev: v0.10.0 + rev: v0.11.0 hooks: - id: shellcheck - repo: local diff --git a/backend/benefit/.prod/uwsgi.ini b/backend/benefit/.prod/uwsgi.ini index c0127b2cc0..7931f0fe6b 100644 --- a/backend/benefit/.prod/uwsgi.ini +++ b/backend/benefit/.prod/uwsgi.ini @@ -6,10 +6,13 @@ module = helsinkibenefit.wsgi static-map = /static=/var/static uid = nobody gid = nogroup +enable-threads = true +py-call-uwsgi-fork-hooks = true buffer-size = 32768 master = 1 processes = 2 threads = 2 +thunder-lock = true ; don't log readiness and healthz endpoints route = ^/readiness$ donotlog: route = ^/healthz$ donotlog: @@ -23,4 +26,18 @@ log-req-encoder = nl # http pipes are closed before workers has had the time to serve content to the pipe ignore-sigpipe = true ignore-write-errors = true -disable-write-exception = true \ No newline at end of file +disable-write-exception = true + +plugin-dir = /usr/local/lib/uwsgi/plugins + +# Sentry logging for uWSGI +if-env = ENABLE_SENTRY_UWSGI_PLUGIN +print = Enabled sentry logging for uWSGI +plugin = sentry +alarm = logsentry sentry:dsn=$(SENTRY_DSN),logger=uwsgi.sentry + +# Log full queue, segfault and harakiri errors to Sentry +alarm-backlog = logsentry +alarm-segfault = logsentry +alarm-log = logsentry HARAKIRI \[core.*\] +endif = diff --git a/backend/benefit/helsinkibenefit/settings.py b/backend/benefit/helsinkibenefit/settings.py index 34b530a511..36b2226ac9 100644 --- a/backend/benefit/helsinkibenefit/settings.py +++ b/backend/benefit/helsinkibenefit/settings.py @@ -5,6 +5,7 @@ from corsheaders.defaults import default_headers from django.utils.translation import gettext_lazy as _ from sentry_sdk.integrations.django import DjangoIntegration +from sentry_sdk.types import SamplingContext from shared.service_bus.enums import YtjOrganizationCode @@ -39,13 +40,16 @@ MAIL_MAILGUN_DOMAIN=(str, ""), MAIL_MAILGUN_API=(str, ""), SENTRY_DSN=(str, ""), - SENTRY_ENVIRONMENT=(str, ""), + SENTRY_ENVIRONMENT=(str, "local"), + SENTRY_PROFILE_SESSION_SAMPLE_RATE=(float, None), + SENTRY_RELEASE=(str, None), + SENTRY_TRACES_SAMPLE_RATE=(float, None), + SENTRY_TRACES_IGNORE_PATHS=(list, ["/healthz", "/readiness"]), SENTRY_ATTACH_STACKTRACE=(bool, False), SENTRY_MAX_BREADCRUMBS=(int, 0), SENTRY_MAX_REQUEST_BODY_SIZE=(str, "never"), SENTRY_SEND_DEFAULT_PII=(bool, False), SENTRY_INCLUDE_LOCAL_VARIABLES=(bool, False), - SENTRY_RELEASE=(str, ""), CORS_ALLOWED_ORIGINS=(list, []), CORS_ALLOW_ALL_ORIGINS=(bool, False), CSRF_COOKIE_DOMAIN=(str, "localhost"), @@ -222,17 +226,42 @@ CACHES = {"default": env.cache()} -sentry_sdk.init( - attach_stacktrace=env.bool("SENTRY_ATTACH_STACKTRACE"), - max_breadcrumbs=env.int("SENTRY_MAX_BREADCRUMBS"), - max_request_body_size=env.str("SENTRY_MAX_REQUEST_BODY_SIZE"), - send_default_pii=env.bool("SENTRY_SEND_DEFAULT_PII"), - include_local_variables=env.bool("SENTRY_INCLUDE_LOCAL_VARIABLES"), - dsn=env.str("SENTRY_DSN"), - release=env.str("SENTRY_RELEASE"), - environment=env.str("SENTRY_ENVIRONMENT"), - integrations=[DjangoIntegration()], -) +# Sentry configuration +SENTRY_TRACES_SAMPLE_RATE = env("SENTRY_TRACES_SAMPLE_RATE") +SENTRY_TRACES_IGNORE_PATHS = env.list("SENTRY_TRACES_IGNORE_PATHS") + + +def sentry_traces_sampler(sampling_context: SamplingContext) -> float: + # Respect parent sampling decision if one exists. Recommended by Sentry. + if (parent_sampled := sampling_context.get("parent_sampled")) is not None: + return float(parent_sampled) + + # Exclude health check endpoints from tracing + path = sampling_context.get("wsgi_environ", {}).get("PATH_INFO", "") + if path.rstrip("/") in SENTRY_TRACES_IGNORE_PATHS: + return 0 + + # Use configured sample rate for all other requests + return SENTRY_TRACES_SAMPLE_RATE or 0 + + +SENTRY_DSN = env("SENTRY_DSN") + +if SENTRY_DSN: + sentry_sdk.init( + attach_stacktrace=env.bool("SENTRY_ATTACH_STACKTRACE"), + max_breadcrumbs=env.int("SENTRY_MAX_BREADCRUMBS"), + max_request_body_size=env.str("SENTRY_MAX_REQUEST_BODY_SIZE"), + send_default_pii=env.bool("SENTRY_SEND_DEFAULT_PII"), + include_local_variables=env.bool("SENTRY_INCLUDE_LOCAL_VARIABLES"), + dsn=SENTRY_DSN, + environment=env("SENTRY_ENVIRONMENT"), + release=env("SENTRY_RELEASE"), + integrations=[DjangoIntegration()], + traces_sampler=sentry_traces_sampler, + profile_session_sample_rate=env("SENTRY_PROFILE_SESSION_SAMPLE_RATE"), + profile_lifecycle="trace", + ) MEDIA_ROOT = env("MEDIA_ROOT") STATIC_ROOT = env("STATIC_ROOT") diff --git a/backend/benefit/helsinkibenefit/tests/test_settings.py b/backend/benefit/helsinkibenefit/tests/test_settings.py new file mode 100644 index 0000000000..e799af7b62 --- /dev/null +++ b/backend/benefit/helsinkibenefit/tests/test_settings.py @@ -0,0 +1,107 @@ +import os +from unittest.mock import patch + + +class TestSentryConfiguration: + """Tests for Sentry configuration and traces sampler.""" + + def test_sentry_traces_sampler_with_parent_sampled_true(self): + """Test that parent sampling decision is respected when True.""" + from helsinkibenefit.settings import sentry_traces_sampler + + sampling_context = {"parent_sampled": True} + result = sentry_traces_sampler(sampling_context) + + assert str(result) == str(1.0) + + def test_sentry_traces_sampler_with_parent_sampled_false(self): + """Test that parent sampling decision is respected when False.""" + from helsinkibenefit.settings import sentry_traces_sampler + + sampling_context = {"parent_sampled": False} + result = sentry_traces_sampler(sampling_context) + + assert str(result) == str(0.0) + + @patch( + "helsinkibenefit.settings.SENTRY_TRACES_IGNORE_PATHS", + ["/healthz", "/readiness"], + ) + def test_sentry_traces_sampler_ignores_health_check_paths(self): + """Test that health check endpoints return 0 sample rate.""" + from helsinkibenefit.settings import sentry_traces_sampler + + sampling_context = { + "parent_sampled": None, + "wsgi_environ": {"PATH_INFO": "/healthz/"}, + } + result = sentry_traces_sampler(sampling_context) + + assert result == 0 + + @patch("helsinkibenefit.settings.SENTRY_TRACES_SAMPLE_RATE", 0.5) + def test_sentry_traces_sampler_uses_configured_rate(self): + """Test that configured sample rate is used for normal requests.""" + from helsinkibenefit.settings import sentry_traces_sampler + + sampling_context = { + "parent_sampled": None, + "wsgi_environ": {"PATH_INFO": "/api/v1/applications/"}, + } + result = sentry_traces_sampler(sampling_context) + + assert str(result) == str(0.5) + + @patch("helsinkibenefit.settings.SENTRY_TRACES_SAMPLE_RATE", None) + def test_sentry_traces_sampler_defaults_to_zero(self): + """Test that sample rate defaults to 0 when not configured.""" + from helsinkibenefit.settings import sentry_traces_sampler + + sampling_context = { + "parent_sampled": None, + "wsgi_environ": {"PATH_INFO": "/api/v1/applications/"}, + } + result = sentry_traces_sampler(sampling_context) + + assert result == 0 + + @patch.dict( + os.environ, {"SENTRY_DSN": "https://example@sentry.io/123456"}, clear=False + ) + @patch("sentry_sdk.init") + def test_sentry_init_called_with_dsn(self, mock_sentry_init): + """Test that Sentry is initialized when DSN is configured.""" + import sys + + # Remove the module from cache to force reimport + if "helsinkibenefit.settings" in sys.modules: + del sys.modules["helsinkibenefit.settings"] + + # Import will trigger the module-level sentry_sdk.init call + import helsinkibenefit.settings # noqa: F401 + + # Verify sentry_sdk.init was called + assert mock_sentry_init.called + # Optionally verify it was called with correct DSN + assert any( + "https://example@sentry.io/123456" in str(call_args) + for call_args in mock_sentry_init.call_args_list + ) + + @patch.dict(os.environ, {"SENTRY_DSN": ""}, clear=False) + @patch("sentry_sdk.init") + def test_sentry_init_not_called_without_dsn(self, mock_sentry_init): + """Test that Sentry is not initialized when DSN is empty.""" + import sys + + # Remove the module from cache to force reimport + if "helsinkibenefit.settings" in sys.modules: + del sys.modules["helsinkibenefit.settings"] + + # Reset the mock to clear any previous calls + mock_sentry_init.reset_mock() + + # Import with empty DSN should not call sentry_sdk.init + import helsinkibenefit.settings # noqa: F401 + + assert not mock_sentry_init.called diff --git a/backend/benefit/pyproject.toml b/backend/benefit/pyproject.toml index d473019fe8..eb0827ba47 100644 --- a/backend/benefit/pyproject.toml +++ b/backend/benefit/pyproject.toml @@ -24,5 +24,6 @@ extend-per-file-ignores = { "*/migrations/*" = ["E501"], "*/tests/*" = ["E501"] [tool.ruff.lint.isort] known-first-party = ["shared"] -[tool.pip-tools] +[tool.pip-tools.compile] strip-extras = true +allow-unsafe = true diff --git a/backend/benefit/requirements-dev.txt b/backend/benefit/requirements-dev.txt index 8cba02d3d7..62b53e88cb 100644 --- a/backend/benefit/requirements-dev.txt +++ b/backend/benefit/requirements-dev.txt @@ -2,23 +2,23 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --strip-extras requirements-dev.in +# pip-compile --allow-unsafe --strip-extras requirements-dev.in # -asttokens==3.0.0 +asttokens==3.0.1 # via stack-data build==1.3.0 # via pip-tools -certifi==2025.8.3 +certifi==2025.11.12 # via # -c requirements.txt # requests -charset-normalizer==3.4.3 +charset-normalizer==3.4.4 # via # -c requirements.txt # requests -click==8.3.0 +click==8.3.1 # via pip-tools -coverage==7.10.7 +coverage==7.12.0 # via pytest-cov decorator==5.2.1 # via ipython @@ -26,13 +26,13 @@ executing==2.2.1 # via stack-data freezegun==1.5.5 # via pytest-freezer -idna==3.10 +idna==3.11 # via # -c requirements.txt # requests -iniconfig==2.1.0 +iniconfig==2.3.0 # via pytest -ipython==9.6.0 +ipython==9.7.0 # via -r requirements-dev.in ipython-pygments-lexers==1.1.1 # via ipython @@ -40,11 +40,11 @@ jedi==0.19.2 # via ipython langdetect==1.0.9 # via -r requirements-dev.in -markupsafe==3.0.2 +markupsafe==3.0.3 # via # -c requirements.txt # werkzeug -matplotlib-inline==0.1.7 +matplotlib-inline==0.2.1 # via ipython packaging==25.0 # via @@ -55,7 +55,7 @@ parso==0.8.5 # via jedi pexpect==4.9.0 # via ipython -pip-tools==7.5.1 +pip-tools==7.5.2 # via -r requirements-dev.in pluggy==1.6.0 # via @@ -76,7 +76,7 @@ pyproject-hooks==1.2.0 # via # build # pip-tools -pytest==8.4.2 +pytest==9.0.1 # via # -r requirements-dev.in # pytest-cov @@ -121,5 +121,7 @@ wheel==0.45.1 # via pip-tools # The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools +pip==25.3 + # via pip-tools +setuptools==80.9.0 + # via pip-tools diff --git a/backend/benefit/requirements-prod.txt b/backend/benefit/requirements-prod.txt index 963579b8e6..20cf0d6993 100644 --- a/backend/benefit/requirements-prod.txt +++ b/backend/benefit/requirements-prod.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --strip-extras requirements-prod.in +# pip-compile --allow-unsafe --strip-extras requirements-prod.in # -uwsgi==2.0.30 +uwsgi==2.0.31 # via -r requirements-prod.in diff --git a/backend/benefit/requirements.in b/backend/benefit/requirements.in index 99b64c8e6f..3aa2f8a914 100644 --- a/backend/benefit/requirements.in +++ b/backend/benefit/requirements.in @@ -33,5 +33,5 @@ python-stdnum>=1.19 python-Levenshtein pyyaml requests -sentry-sdk +sentry-sdk[django] uritemplate diff --git a/backend/benefit/requirements.txt b/backend/benefit/requirements.txt index ff7c8b5f24..846b7a3385 100644 --- a/backend/benefit/requirements.txt +++ b/backend/benefit/requirements.txt @@ -2,38 +2,38 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile --strip-extras requirements.in +# pip-compile --allow-unsafe --strip-extras requirements.in # -e file:../shared # via -r requirements.in -asgiref==3.9.1 +asgiref==3.11.0 # via # django # django-cors-headers -attrs==25.3.0 +attrs==25.4.0 # via # jsonschema # referencing authlib==1.6.5 # via drf-oidc-auth -azure-core==1.35.1 +azure-core==1.36.0 # via # azure-storage-blob # django-storages -azure-storage-blob==12.26.0 +azure-storage-blob==12.27.1 # via django-storages babel==2.17.0 # via -r requirements.in -cachetools==6.2.0 +cachetools==6.2.2 # via django-helusers -certifi==2025.8.3 +certifi==2025.11.12 # via # elastic-transport # requests # sentry-sdk cffi==2.0.0 # via cryptography -charset-normalizer==3.4.3 +charset-normalizer==3.4.4 # via requests cryptography==43.0.3 # via @@ -72,8 +72,9 @@ django==5.2.8 # drf-spectacular # helsinki-profile-gdpr-api # mozilla-django-oidc + # sentry-sdk # yjdh-backend-shared -django-auth-adfs==1.15.0 +django-auth-adfs==1.15.1 # via # -r requirements.in # yjdh-backend-shared @@ -85,13 +86,13 @@ django-extensions==4.1 # via # -r requirements.in # yjdh-backend-shared -django-filter==25.1 +django-filter==25.2 # via -r requirements.in -django-helusers==0.14.0 +django-helusers==0.14.4 # via helsinki-profile-gdpr-api django-localflavor==5.0 # via -r requirements.in -django-phonenumber-field==8.1.0 +django-phonenumber-field==8.3.0 # via -r requirements.in django-searchable-encrypted-fields==0.2.1 # via -r requirements.in @@ -114,13 +115,13 @@ drf-nested-routers==0.95.0 # via -r requirements.in drf-oidc-auth==3.0.0 # via helsinki-profile-gdpr-api -drf-spectacular==0.28.0 +drf-spectacular==0.29.0 # via -r requirements.in ecdsa==0.19.1 # via python-jose elastic-transport==8.17.1 # via elasticsearch -elasticsearch==8.19.1 +elasticsearch==8.19.2 # via -r requirements.in elementpath==4.8.0 # via xmlschema @@ -128,7 +129,7 @@ et-xmlfile==2.0.0 # via openpyxl factory-boy==3.3.3 # via -r requirements.in -faker==37.8.0 +faker==38.2.0 # via factory-boy filetype==1.2.0 # via -r requirements.in @@ -136,7 +137,7 @@ fuzzywuzzy==0.18.0 # via -r requirements.in helsinki-profile-gdpr-api==0.2.0 # via -r requirements.in -idna==3.10 +idna==3.11 # via requests inflection==0.5.1 # via drf-spectacular @@ -144,37 +145,37 @@ isodate==0.7.2 # via azure-storage-blob jinja2==3.1.6 # via -r requirements.in -josepy==2.1.0 +josepy==2.2.0 # via mozilla-django-oidc jsonschema==4.25.1 # via drf-spectacular jsonschema-specifications==2025.9.1 # via jsonschema -levenshtein==0.27.1 +levenshtein==0.27.3 # via python-levenshtein lxml==6.0.2 # via -r requirements.in -markupsafe==3.0.2 +markupsafe==3.0.3 # via jinja2 mozilla-django-oidc==4.0.1 # via # -r requirements.in # yjdh-backend-shared -numpy==2.3.3 +numpy==2.3.5 # via pandas openpyxl==3.1.5 # via -r requirements.in packaging==25.0 # via deprecation -pandas==2.3.2 +pandas==2.3.3 # via -r requirements.in pdfkit==1.0.0 # via -r requirements.in -phonenumbers==9.0.14 +phonenumbers==9.0.19 # via django-phonenumber-field -pillow==11.3.0 +pillow==12.0.0 # via -r requirements.in -psycopg2==2.9.10 +psycopg2==2.9.11 # via -r requirements.in pyasn1==0.6.1 # via @@ -188,7 +189,7 @@ pyjwt==2.10.1 # via django-auth-adfs pyopenssl==24.2.1 # via pysaml2 -pysaml2==7.5.2 +pysaml2==7.5.4 # via djangosaml2 python-dateutil==2.9.0.post0 # via @@ -198,23 +199,21 @@ python-dateutil==2.9.0.post0 # pysaml2 python-jose==3.5.0 # via django-helusers -python-levenshtein==0.27.1 +python-levenshtein==0.27.3 # via -r requirements.in python-stdnum==2.1 # via # -r requirements.in # django-localflavor pytz==2025.2 - # via - # pandas - # pysaml2 -pyyaml==6.0.2 + # via pandas +pyyaml==6.0.3 # via # -r requirements.in # drf-spectacular -rapidfuzz==3.14.1 +rapidfuzz==3.14.3 # via levenshtein -referencing==0.36.2 +referencing==0.37.0 # via # jsonschema # jsonschema-specifications @@ -227,17 +226,16 @@ requests==2.32.5 # drf-oidc-auth # mozilla-django-oidc # pysaml2 -rpds-py==0.27.1 +rpds-py==0.29.0 # via # jsonschema # referencing rsa==4.9.1 # via python-jose -sentry-sdk==2.38.0 +sentry-sdk==2.46.0 # via -r requirements.in six==1.17.0 # via - # azure-core # ecdsa # python-dateutil sqlparse==0.5.3 diff --git a/backend/docker/benefit-ubi-arm.Dockerfile b/backend/docker/benefit-ubi-arm.Dockerfile index bebc40bdac..f755821f09 100644 --- a/backend/docker/benefit-ubi-arm.Dockerfile +++ b/backend/docker/benefit-ubi-arm.Dockerfile @@ -29,6 +29,9 @@ RUN dnf update -y \ && pip install --no-cache-dir -r /app/requirements-prod.txt \ && uwsgi --build-plugin /app/.prod/escape_json.c \ && mv /app/escape_json_plugin.so /app/.prod/escape_json_plugin.so \ + && mkdir -p /usr/local/lib/uwsgi/plugins \ + && uwsgi --build-plugin https://github.com/City-of-Helsinki/uwsgi-sentry \ + && mv sentry_plugin.so /usr/local/lib/uwsgi/plugins/ \ && dnf remove -y gcc cyrus-sasl-devel openssl-devel # Install wkhtmltopdf and it's deps from CentOS9 repo and binary diff --git a/compose.benefit-backend.yml b/compose.benefit-backend.yml index c55557f997..943f4bffd6 100644 --- a/compose.benefit-backend.yml +++ b/compose.benefit-backend.yml @@ -57,6 +57,7 @@ services: clamav: image: clamav/clamav:latest + platform: linux/amd64 container_name: benefit-clamav environment: CLAMAV_NO_LOGFILE: "true" @@ -70,6 +71,7 @@ services: clamav-rest-api: image: benzino77/clamav-rest-api + platform: linux/amd64 container_name: benefit-clamav-rest-api environment: NODE_ENV: production