From 49fb9aa1bb3d5b7dd685e803c1dd71c6c09974e0 Mon Sep 17 00:00:00 2001 From: jeriox Date: Mon, 11 May 2026 22:10:02 +0200 Subject: [PATCH 1/3] fix db deadlocks --- pyproject.toml | 5 +- src/ephios/modellogging/json.py | 17 ++++--- ...6_convert_contenttype_id_to_natural_key.py | 50 +++++++++++++++++++ src/ephios/modellogging/recorders.py | 4 +- uv.lock | 43 ++++++++++++---- 5 files changed, 99 insertions(+), 20 deletions(-) create mode 100644 src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py diff --git a/pyproject.toml b/pyproject.toml index eb4f2d726..3f8092258 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,9 @@ dependencies = [ ] [project.optional-dependencies] -pgsql = ["psycopg2>=2.9.3,<3"] +pgsql = [ + "psycopg>=3.3.4", +] mysql = ["mysqlclient>=2.1.1,<3"] redis = ["redis[hiredis]>=5,<8"] @@ -77,6 +79,7 @@ dev = [ "djhtml>=3.0.6,<4", "prek>=0.3.1", "ruff>=0.15.0", + "pytest-xdist>=3.8.0", ] [project.urls] diff --git a/src/ephios/modellogging/json.py b/src/ephios/modellogging/json.py index 4c07e19fb..93f3b2be5 100644 --- a/src/ephios/modellogging/json.py +++ b/src/ephios/modellogging/json.py @@ -29,6 +29,7 @@ class LogJSONEncoder(DjangoJSONEncoder): def default(self, o): if isinstance(o, QuerySet) or _is_queryset_like(o): + model = getattr(o, "model", None) or type(next(iter(o))) pks, strs = [], [] for instance in o: pks.append(instance.pk) @@ -37,9 +38,8 @@ def default(self, o): "__model__": "__queryset__", "pks": pks, "strs": strs, - "contenttype_id": ContentType.objects.get_for_model( - getattr(o, "model", None) or type(next(iter(o))) - ).id, + "app_label": model._meta.app_label, + "model": model._meta.model_name, } if o == set(): return [] @@ -48,7 +48,8 @@ def default(self, o): "__model__": "__instance__", "pk": o.pk, "str": str(o), - "contenttype_id": ContentType.objects.get_for_model(o).id, + "app_label": o._meta.app_label, + "model": o._meta.model_name, } return super().default(o) @@ -63,16 +64,16 @@ def __init__(self, *args, **kargs): def custom_hook(self, d): if d.get("__model__") == "__queryset__": - Model = ContentType.objects.get_for_id(d["contenttype_id"]).model_class() + Model = ContentType.objects.get_by_natural_key(d["app_label"], d["model"]).model_class() if Model is None: return d["strs"] objects = {obj.pk: obj for obj in Model._base_manager.filter(pk__in=d["pks"])} return [objects.get(pk, s) for pk, s in zip(d["pks"], d["strs"])] if d.get("__model__") == "__instance__": try: - return ContentType.objects.get_for_id(d["contenttype_id"]).get_object_for_this_type( - pk=d["pk"] - ) + return ContentType.objects.get_by_natural_key( + d["app_label"], d["model"] + ).get_object_for_this_type(pk=d["pk"]) except (ObjectDoesNotExist, AttributeError): return d["str"] for k, v in d.items(): diff --git a/src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py b/src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py new file mode 100644 index 000000000..4e0c7f3f8 --- /dev/null +++ b/src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py @@ -0,0 +1,50 @@ +from django.db import migrations + + +def convert_contenttype_id_to_natural_key(apps, schema_editor): + LogEntry = apps.get_model("modellogging", "LogEntry") + # our custom JSONDecoder has been changed to the new format, so it fails to read the existing entries + LogEntry._meta.get_field("data").decoder = None + ContentType = apps.get_model("contenttypes", "ContentType") + ct_cache = {} + + for entry in LogEntry.objects.all(): + changed = False + data = entry.data + if not isinstance(data, dict): + continue + for obj in _find_model_refs(data): + if "contenttype_id" in obj and "app_label" not in obj: + ct_id = obj["contenttype_id"] + if ct_id not in ct_cache: + ct = ContentType.objects.get(pk=ct_id) + ct_cache[ct_id] = (ct.app_label, ct.model) + obj["app_label"], obj["model"] = ct_cache[ct_id] + changed = True + if changed: + entry.data = data + entry.save() + + +def _find_model_refs(d): + if isinstance(d, dict): + if d.get("__model__") in ("__instance__", "__queryset__"): + yield d + for v in d.values(): + yield from _find_model_refs(v) + elif isinstance(d, list): + for item in d: + yield from _find_model_refs(item) + + +class Migration(migrations.Migration): + dependencies = [ + ("modellogging", "0005_alter_logentry_datetime"), + ] + + operations = [ + migrations.RunPython( + convert_contenttype_id_to_natural_key, + migrations.RunPython.noop, + ), + ] diff --git a/src/ephios/modellogging/recorders.py b/src/ephios/modellogging/recorders.py index 144a256f5..4013c01d1 100644 --- a/src/ephios/modellogging/recorders.py +++ b/src/ephios/modellogging/recorders.py @@ -251,8 +251,8 @@ def serialize(self, action_type: InstanceActionType): data = { "field_name": self.field_name, "verbose_name": self.verbose_name, - "added": related_model._base_manager.filter(pk__in=self.added_pks), - "removed": related_model._base_manager.filter(pk__in=self.removed_pks), + "added": list(related_model._base_manager.filter(pk__in=self.added_pks)), + "removed": list(related_model._base_manager.filter(pk__in=self.removed_pks)), } if (current := getattr(self, "current", None)) is not None: diff --git a/uv.lock b/uv.lock index cc27f5906..e681bf6e5 100644 --- a/uv.lock +++ b/uv.lock @@ -1080,7 +1080,7 @@ mysql = [ { name = "mysqlclient" }, ] pgsql = [ - { name = "psycopg2" }, + { name = "psycopg" }, ] redis = [ { name = "redis", extra = ["hiredis"] }, @@ -1098,6 +1098,7 @@ dev = [ { name = "prek" }, { name = "pylint" }, { name = "pytest-django" }, + { name = "pytest-xdist" }, { name = "ruff" }, { name = "sphinx" }, { name = "sphinx-github-changelog" }, @@ -1135,7 +1136,7 @@ requires-dist = [ { name = "lxml", specifier = ">=4.9.3,<7.0.0" }, { name = "markdown", specifier = ">=3.3.7,<4" }, { name = "mysqlclient", marker = "extra == 'mysql'", specifier = ">=2.1.1,<3" }, - { name = "psycopg2", marker = "extra == 'pgsql'", specifier = ">=2.9.3,<3" }, + { name = "psycopg", marker = "extra == 'pgsql'", specifier = ">=3.3.4" }, { name = "py-vapid", specifier = ">=1.9.4,<2" }, { name = "pyjwt", specifier = ">=2.8.0,<3" }, { name = "python-dateutil", specifier = ">=2.9.0.post0,<3" }, @@ -1163,6 +1164,7 @@ dev = [ { name = "prek", specifier = ">=0.3.1" }, { name = "pylint", specifier = ">=3.0.0,<5.0.0" }, { name = "pytest-django", specifier = ">=4.5.2,<5" }, + { name = "pytest-xdist", specifier = ">=3.8.0" }, { name = "ruff", specifier = ">=0.15.0" }, { name = "sphinx", specifier = ">=8.2,<10.0.0" }, { name = "sphinx-github-changelog", specifier = ">=1.2.0,<2" }, @@ -1170,6 +1172,15 @@ dev = [ { name = "sphinx-rtd-theme", specifier = ">=3,<4" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "frozenlist" version = "1.8.0" @@ -2169,15 +2180,16 @@ wheels = [ ] [[package]] -name = "psycopg2" -version = "2.9.12" +name = "psycopg" +version = "3.3.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c7/bc/f66df707ed1aec949fbf24e4460e4f4277a7ba23cdadb3965bb1f634ddb9/psycopg2-2.9.12.tar.gz", hash = "sha256:1dedb1c7a1d8552c4a6044c6b1c41a52e6a8e2d144af83eccac758076b1b7c15", size = 379683, upload-time = "2026-04-20T23:36:11.013Z" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/db/2f/cb91e5502ec9de1de6f1b76cfbf69531932725361168bb06963620c77e2e/psycopg-3.3.4.tar.gz", hash = "sha256:e21207764952cff81b6b8bdacad9a3939f2793367fdac2987b3aac36a651b5bc", size = 165799, upload-time = "2026-05-01T23:31:55.179Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/a1/9cc069f8efc92383d3823ef8805c932f5fda723b3f07d2a8ee019ae2919f/psycopg2-2.9.12-cp311-cp311-win_amd64.whl", hash = "sha256:2532c0cdc6ad18c9c35cd935cc3159712e14f05276a6d29a6435c52d24b840c1", size = 2756862, upload-time = "2026-04-20T23:33:17.206Z" }, - { url = "https://files.pythonhosted.org/packages/7e/b2/7319dc488444b1dfe9ce89c4440df1d5425e2579e6ce9a63e2b70f648491/psycopg2-2.9.12-cp312-cp312-win_amd64.whl", hash = "sha256:83d48e66e18c301d832e93c984a7bcbc0f4ac3bb79e2137e3bc335978c756dc0", size = 2757068, upload-time = "2026-04-20T23:33:20.664Z" }, - { url = "https://files.pythonhosted.org/packages/91/ab/03403779a251ca14a7f1b07a41c7aa735fd451f0aebe4c4f901a231167a4/psycopg2-2.9.12-cp313-cp313-win_amd64.whl", hash = "sha256:3d23e684927d37b95cee9a943f6927b04ae2fdcd056fd0e2a30929ee89fee5a9", size = 2757176, upload-time = "2026-04-20T23:33:23.577Z" }, - { url = "https://files.pythonhosted.org/packages/73/4a/a3566f77501c21a6c2a1fc234dbe5dff74a86e3f150ef070e4ffb835e7f9/psycopg2-2.9.12-cp314-cp314-win_amd64.whl", hash = "sha256:a73d5513bfe929c56555006c7a9cc7ae6e4276aa99dd2b1e2544eb8bb54f8b23", size = 2848588, upload-time = "2026-04-20T23:33:25.983Z" }, + { url = "https://files.pythonhosted.org/packages/5c/e0/7b3dee031daae7743609ce3c746565d4a3ed7c2c186479eb48e34e838c64/psycopg-3.3.4-py3-none-any.whl", hash = "sha256:b6bbc25ccf05c8fad3b061d9db2ef0909a555171b84b07f29458a447253d679a", size = 213001, upload-time = "2026-05-01T23:20:50.816Z" }, ] [[package]] @@ -2265,6 +2277,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/83/a5/41d091f697c09609e7ef1d5d61925494e0454ebf51de7de05f0f0a728f1d/pytest_django-4.12.0-py3-none-any.whl", hash = "sha256:3ff300c49f8350ba2953b90297d23bf5f589db69545f56f1ec5f8cff5da83e85", size = 26123, upload-time = "2026-02-14T18:40:47.381Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" From ce1126023c0f7720e3d42ec8262e9f37cbf90dc6 Mon Sep 17 00:00:00 2001 From: jeriox Date: Mon, 11 May 2026 22:35:06 +0200 Subject: [PATCH 2/3] fix db deadlocks --- src/ephios/modellogging/log.py | 6 ++++++ src/ephios/modellogging/migrations/0001_initial.py | 5 ++--- .../0006_convert_contenttype_id_to_natural_key.py | 9 ++++++++- src/ephios/modellogging/models.py | 4 ++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/ephios/modellogging/log.py b/src/ephios/modellogging/log.py index 2d52cd875..f942ce4fb 100644 --- a/src/ephios/modellogging/log.py +++ b/src/ephios/modellogging/log.py @@ -1,5 +1,6 @@ import contextvars import itertools +import json from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist @@ -8,6 +9,7 @@ from django.db.models.signals import post_init, post_save, pre_delete, pre_save from django.dispatch import receiver +from ephios.modellogging.json import LogJSONEncoder from ephios.modellogging.models import LogEntry from ephios.modellogging.recorders import ( InstanceActionType, @@ -180,6 +182,10 @@ def update_log(instance, action_type: InstanceActionType): if not log_data: return + # Pre-serialize to resolve model instances, querysets, and str() calls + # outside of psycopg3's connection lock (which would deadlock on DB queries). + log_data = json.loads(json.dumps(log_data, cls=LogJSONEncoder)) + config = LOGGED_MODELS[type(instance)] if logentry: logentry.data.update(log_data) diff --git a/src/ephios/modellogging/migrations/0001_initial.py b/src/ephios/modellogging/migrations/0001_initial.py index f2bdcf70a..8de5da900 100644 --- a/src/ephios/modellogging/migrations/0001_initial.py +++ b/src/ephios/modellogging/migrations/0001_initial.py @@ -4,6 +4,7 @@ from django.conf import settings from django.db import migrations, models +import ephios.modellogging.json import ephios.modellogging.models @@ -51,9 +52,7 @@ class Migration(migrations.Migration): ("request_id", models.CharField(blank=True, max_length=36, null=True)), ( "data", - models.JSONField( - default=dict, encoder=ephios.modellogging.models.LogJSONEncoder - ), + models.JSONField(default=dict, encoder=ephios.modellogging.json.LogJSONEncoder), ), ( "attached_to_object_type", diff --git a/src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py b/src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py index 4e0c7f3f8..59797bc1c 100644 --- a/src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py +++ b/src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py @@ -1,4 +1,6 @@ -from django.db import migrations +from django.db import migrations, models + +import ephios.modellogging.json def convert_contenttype_id_to_natural_key(apps, schema_editor): @@ -47,4 +49,9 @@ class Migration(migrations.Migration): convert_contenttype_id_to_natural_key, migrations.RunPython.noop, ), + migrations.AlterField( + model_name="logentry", + name="data", + field=models.JSONField(decoder=ephios.modellogging.json.LogJSONDecoder, default=dict), + ), ] diff --git a/src/ephios/modellogging/models.py b/src/ephios/modellogging/models.py index 7a3aa188f..6ceedfcb9 100644 --- a/src/ephios/modellogging/models.py +++ b/src/ephios/modellogging/models.py @@ -5,7 +5,7 @@ from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _ -from ephios.modellogging.json import LogJSONDecoder, LogJSONEncoder +from ephios.modellogging.json import LogJSONDecoder from ephios.modellogging.recorders import ( InstanceActionType, capitalize_first, @@ -41,7 +41,7 @@ class LogEntry(models.Model): max_length=255, choices=[(value, value) for value in InstanceActionType] ) request_id = models.CharField(max_length=36, null=True, blank=True) - data = models.JSONField(default=dict, encoder=LogJSONEncoder, decoder=LogJSONDecoder) + data = models.JSONField(default=dict, decoder=LogJSONDecoder) class Meta: ordering = ("-datetime", "-id") From 7c4b5ae274cc5b1c59b65455c6d62655a36186c9 Mon Sep 17 00:00:00 2001 From: jeriox Date: Mon, 11 May 2026 22:39:20 +0200 Subject: [PATCH 3/3] fix db deadlocks --- .github/workflows/tests.yml | 2 +- .../migrations/0006_convert_contenttype_id_to_natural_key.py | 2 ++ src/ephios/modellogging/models.py | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ca5083b06..792e69808 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -86,7 +86,7 @@ jobs: - name: Test apps env: DATABASE_URL: ${{ matrix.database_url }} - run: uv run coverage run -m pytest tests/ + run: uv run coverage run -m pytest tests/ -n 8 - name: Coveralls env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py b/src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py index 59797bc1c..d82029b39 100644 --- a/src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py +++ b/src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py @@ -2,6 +2,8 @@ import ephios.modellogging.json +# pylint: disable=protected-access + def convert_contenttype_id_to_natural_key(apps, schema_editor): LogEntry = apps.get_model("modellogging", "LogEntry") diff --git a/src/ephios/modellogging/models.py b/src/ephios/modellogging/models.py index 6ceedfcb9..0ccbe671c 100644 --- a/src/ephios/modellogging/models.py +++ b/src/ephios/modellogging/models.py @@ -54,6 +54,8 @@ def records(self): for recorder in self.data.values(): if not isinstance(recorder, dict) or "slug" not in recorder: continue + if recorder["slug"] not in recorder_types: + continue yield recorder_types[recorder["slug"]].deserialize( recorder["data"], self.content_type.model_class(), self.action_type )