update psycopg to v3

.github/workflows/tests.yml

@@ -86,7 +86,7 @@ jobs:
     - name: Test apps
       env:
         DATABASE_URL: ${{ matrix.database_url }}
-      run: uv run coverage run -m pytest tests/
+      run: uv run coverage run -m pytest tests/ -n 8
     - name: Coveralls
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

pyproject.toml

@@ -56,7 +56,9 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-pgsql = ["psycopg2>=2.9.3,<3"]
+pgsql = [
+    "psycopg>=3.3.4",
+]
 mysql = ["mysqlclient>=2.1.1,<3"]
 redis = ["redis[hiredis]>=5,<8"]
 
@@ -77,6 +79,7 @@ dev = [
     "djhtml>=3.0.6,<4",
     "prek>=0.3.1",
     "ruff>=0.15.0",
+    "pytest-xdist>=3.8.0",
 ]
 
 [project.urls]

src/ephios/modellogging/json.py

@@ -29,6 +29,7 @@ class LogJSONEncoder(DjangoJSONEncoder):
 
     def default(self, o):
         if isinstance(o, QuerySet) or _is_queryset_like(o):
+            model = getattr(o, "model", None) or type(next(iter(o)))
             pks, strs = [], []
             for instance in o:
                 pks.append(instance.pk)
@@ -37,9 +38,8 @@ def default(self, o):
                 "__model__": "__queryset__",
                 "pks": pks,
                 "strs": strs,
-                "contenttype_id": ContentType.objects.get_for_model(
-                    getattr(o, "model", None) or type(next(iter(o)))
-                ).id,
+                "app_label": model._meta.app_label,
+                "model": model._meta.model_name,
             }
         if o == set():
             return []
@@ -48,7 +48,8 @@ def default(self, o):
                 "__model__": "__instance__",
                 "pk": o.pk,
                 "str": str(o),
-                "contenttype_id": ContentType.objects.get_for_model(o).id,
+                "app_label": o._meta.app_label,
+                "model": o._meta.model_name,
             }
         return super().default(o)
 
@@ -63,16 +64,16 @@ def __init__(self, *args, **kargs):
 
     def custom_hook(self, d):
         if d.get("__model__") == "__queryset__":
-            Model = ContentType.objects.get_for_id(d["contenttype_id"]).model_class()
+            Model = ContentType.objects.get_by_natural_key(d["app_label"], d["model"]).model_class()
             if Model is None:
                 return d["strs"]
             objects = {obj.pk: obj for obj in Model._base_manager.filter(pk__in=d["pks"])}
             return [objects.get(pk, s) for pk, s in zip(d["pks"], d["strs"])]
         if d.get("__model__") == "__instance__":
             try:
-                return ContentType.objects.get_for_id(d["contenttype_id"]).get_object_for_this_type(
-                    pk=d["pk"]
-                )
+                return ContentType.objects.get_by_natural_key(
+                    d["app_label"], d["model"]
+                ).get_object_for_this_type(pk=d["pk"])
             except (ObjectDoesNotExist, AttributeError):
                 return d["str"]
         for k, v in d.items():

src/ephios/modellogging/log.py

@@ -1,5 +1,6 @@
 import contextvars
 import itertools
+import json
 
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ObjectDoesNotExist
@@ -8,6 +9,7 @@
 from django.db.models.signals import post_init, post_save, pre_delete, pre_save
 from django.dispatch import receiver
 
+from ephios.modellogging.json import LogJSONEncoder
 from ephios.modellogging.models import LogEntry
 from ephios.modellogging.recorders import (
     InstanceActionType,
@@ -180,6 +182,10 @@ def update_log(instance, action_type: InstanceActionType):
     if not log_data:
         return
 
+    # Pre-serialize to resolve model instances, querysets, and str() calls
+    # outside of psycopg3's connection lock (which would deadlock on DB queries).
+    log_data = json.loads(json.dumps(log_data, cls=LogJSONEncoder))
+
     config = LOGGED_MODELS[type(instance)]
     if logentry:
         logentry.data.update(log_data)

src/ephios/modellogging/migrations/0001_initial.py

@@ -4,6 +4,7 @@
 from django.conf import settings
 from django.db import migrations, models
 
+import ephios.modellogging.json
 import ephios.modellogging.models
 
 
@@ -51,9 +52,7 @@ class Migration(migrations.Migration):
                 ("request_id", models.CharField(blank=True, max_length=36, null=True)),
                 (
                     "data",
-                    models.JSONField(
-                        default=dict, encoder=ephios.modellogging.models.LogJSONEncoder
-                    ),
+                    models.JSONField(default=dict, encoder=ephios.modellogging.json.LogJSONEncoder),
                 ),
                 (
                     "attached_to_object_type",

src/ephios/modellogging/migrations/0006_convert_contenttype_id_to_natural_key.py

@@ -0,0 +1,59 @@
+from django.db import migrations, models
+
+import ephios.modellogging.json
+
+# pylint: disable=protected-access
+
+
+def convert_contenttype_id_to_natural_key(apps, schema_editor):
+    LogEntry = apps.get_model("modellogging", "LogEntry")
+    # our custom JSONDecoder has been changed to the new format, so it fails to read the existing entries
+    LogEntry._meta.get_field("data").decoder = None
+    ContentType = apps.get_model("contenttypes", "ContentType")
+    ct_cache = {}
+
+    for entry in LogEntry.objects.all():
+        changed = False
+        data = entry.data
+        if not isinstance(data, dict):
+            continue
+        for obj in _find_model_refs(data):
+            if "contenttype_id" in obj and "app_label" not in obj:
+                ct_id = obj["contenttype_id"]
+                if ct_id not in ct_cache:
+                    ct = ContentType.objects.get(pk=ct_id)
+                    ct_cache[ct_id] = (ct.app_label, ct.model)
+                obj["app_label"], obj["model"] = ct_cache[ct_id]
+                changed = True
+        if changed:
+            entry.data = data
+            entry.save()
+
+
+def _find_model_refs(d):
+    if isinstance(d, dict):
+        if d.get("__model__") in ("__instance__", "__queryset__"):
+            yield d
+        for v in d.values():
+            yield from _find_model_refs(v)
+    elif isinstance(d, list):
+        for item in d:
+            yield from _find_model_refs(item)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("modellogging", "0005_alter_logentry_datetime"),
+    ]
+
+    operations = [
+        migrations.RunPython(
+            convert_contenttype_id_to_natural_key,
+            migrations.RunPython.noop,
+        ),
+        migrations.AlterField(
+            model_name="logentry",
+            name="data",
+            field=models.JSONField(decoder=ephios.modellogging.json.LogJSONDecoder, default=dict),
+        ),
+    ]

src/ephios/modellogging/models.py

@@ -5,7 +5,7 @@
 from django.utils.functional import cached_property
 from django.utils.translation import gettext_lazy as _
 
-from ephios.modellogging.json import LogJSONDecoder, LogJSONEncoder
+from ephios.modellogging.json import LogJSONDecoder
 from ephios.modellogging.recorders import (
     InstanceActionType,
     capitalize_first,
@@ -41,7 +41,7 @@ class LogEntry(models.Model):
         max_length=255, choices=[(value, value) for value in InstanceActionType]
     )
     request_id = models.CharField(max_length=36, null=True, blank=True)
-    data = models.JSONField(default=dict, encoder=LogJSONEncoder, decoder=LogJSONDecoder)
+    data = models.JSONField(default=dict, decoder=LogJSONDecoder)
 
     class Meta:
         ordering = ("-datetime", "-id")
@@ -54,6 +54,8 @@ def records(self):
         for recorder in self.data.values():
             if not isinstance(recorder, dict) or "slug" not in recorder:
                 continue
+            if recorder["slug"] not in recorder_types:
+                continue
             yield recorder_types[recorder["slug"]].deserialize(
                 recorder["data"], self.content_type.model_class(), self.action_type
             )

src/ephios/modellogging/recorders.py

@@ -251,8 +251,8 @@ def serialize(self, action_type: InstanceActionType):
         data = {
             "field_name": self.field_name,
             "verbose_name": self.verbose_name,
-            "added": related_model._base_manager.filter(pk__in=self.added_pks),
-            "removed": related_model._base_manager.filter(pk__in=self.removed_pks),
+            "added": list(related_model._base_manager.filter(pk__in=self.added_pks)),
+            "removed": list(related_model._base_manager.filter(pk__in=self.removed_pks)),
         }
 
         if (current := getattr(self, "current", None)) is not None: