Coverage for ivatar/ivataraccount/migrations/0025_access_stat_tables.py: 32%
56 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-21 11:51 +0000
1"""
2Migration: Move access_count from parent tables to dedicated stat tables.
4Creates PhotoAccessStat, ConfirmedEmailAccessStat, and ConfirmedOpenIdAccessStat,
5copies existing access_count data from the parent tables, then removes the
6access_count columns and their indexes from the parent tables.
8This improves PostgreSQL HOT (Heap Only Tuple) update rates and reduces dead
9tuple accumulation on the wide parent tables.
10"""
12from django.db import migrations, models
13import django.db.models.deletion
16def _flush_cache_key_to_db(model_class, model_name, pk):
17 """
18 Drain any cache-buffered access count for one object into its legacy DB column.
20 Mirrors AccessStatsManager._flush_key_sync exactly — same key format, same
21 distributed lock pattern — so it safely co-exists with any web worker that
22 may be running a concurrent flush at migration time. Writing to the old DB
23 column here (before RemoveField) is intentional: copy_access_counts reads
24 those columns immediately afterwards to populate the stat tables.
25 """
26 from django.core.cache import cache
27 from django.db.models import F
29 key = f"stats:access_count:{model_name}:{pk}"
30 lock_key = f"{key}:lock"
32 # Acquire the same distributed lock used by the web workers.
33 # If we cannot acquire it, a worker is mid-flush; that flush will land on the
34 # old column while it still exists, so copy_access_counts will capture it.
35 if not cache.add(lock_key, "locked", 60):
36 return
38 try:
39 cached = cache.get(key)
40 if not cached:
41 return
42 try:
43 count = int(cached)
44 except (ValueError, TypeError):
45 return
46 if count <= 0:
47 return
49 model_class.objects.filter(pk=pk).update(
50 access_count=F("access_count") + count
51 )
52 try:
53 cache.decr(key, count)
54 except ValueError:
55 cache.set(key, 0)
56 cache.delete(f"stats:start_time:{model_name}:{pk}")
57 except Exception:
58 pass
59 finally:
60 cache.delete(lock_key)
63def copy_access_counts(apps, schema_editor):
64 """
65 Step 1 — drain any cache-buffered counts to the old parent-table columns so
66 that no pending increments are lost when those columns are dropped.
67 Step 2 — bulk-copy the now-complete DB values into the new stat tables.
69 Both steps run inside the migration transaction, so the copy always sees the
70 flushed values. Any count that was being flushed concurrently by a web worker
71 is protected by the distributed lock in _flush_cache_key_to_db; if the lock is
72 held, the worker's own flush will complete to the old column (which still exists
73 at this point in the migration) and the subsequent SELECT in Step 2 will capture
74 it under READ COMMITTED semantics.
75 """
76 Photo = apps.get_model("ivataraccount", "Photo")
77 ConfirmedEmail = apps.get_model("ivataraccount", "ConfirmedEmail")
78 ConfirmedOpenId = apps.get_model("ivataraccount", "ConfirmedOpenId")
79 PhotoAccessStat = apps.get_model("ivataraccount", "PhotoAccessStat")
80 ConfirmedEmailAccessStat = apps.get_model("ivataraccount", "ConfirmedEmailAccessStat")
81 ConfirmedOpenIdAccessStat = apps.get_model("ivataraccount", "ConfirmedOpenIdAccessStat")
83 for model_class, model_name, StatClass, fk_field in (
84 (Photo, "Photo", PhotoAccessStat, "photo_id"),
85 (ConfirmedEmail, "ConfirmedEmail", ConfirmedEmailAccessStat, "email_id"),
86 (ConfirmedOpenId, "ConfirmedOpenId", ConfirmedOpenIdAccessStat, "openid_id"),
87 ):
88 pks = list(model_class.objects.values_list("pk", flat=True))
90 # Step 1: drain pending cache counts into the old DB column.
91 for pk in pks:
92 _flush_cache_key_to_db(model_class, model_name, pk)
94 # Step 2: copy the complete DB values (post-flush) to the stat table.
95 StatClass.objects.bulk_create(
96 [
97 StatClass(**{fk_field: obj.pk, "access_count": obj.access_count})
98 for obj in model_class.objects.only("pk", "access_count")
99 ],
100 ignore_conflicts=True,
101 )
104def reverse_copy_access_counts(apps, schema_editor):
105 """
106 Reverse migration: copy access_count values back to the parent tables.
107 """
108 Photo = apps.get_model("ivataraccount", "Photo")
109 ConfirmedEmail = apps.get_model("ivataraccount", "ConfirmedEmail")
110 ConfirmedOpenId = apps.get_model("ivataraccount", "ConfirmedOpenId")
111 PhotoAccessStat = apps.get_model("ivataraccount", "PhotoAccessStat")
112 ConfirmedEmailAccessStat = apps.get_model("ivataraccount", "ConfirmedEmailAccessStat")
113 ConfirmedOpenIdAccessStat = apps.get_model("ivataraccount", "ConfirmedOpenIdAccessStat")
115 for stat in PhotoAccessStat.objects.all():
116 Photo.objects.filter(pk=stat.photo_id).update(access_count=stat.access_count)
118 for stat in ConfirmedEmailAccessStat.objects.all():
119 ConfirmedEmail.objects.filter(pk=stat.email_id).update(access_count=stat.access_count)
121 for stat in ConfirmedOpenIdAccessStat.objects.all():
122 ConfirmedOpenId.objects.filter(pk=stat.openid_id).update(access_count=stat.access_count)
125class Migration(migrations.Migration):
127 dependencies = [
128 ("ivataraccount", "0024_merge_20260722_1534"),
129 ]
131 operations = [
132 # 1. Create the three new stat tables
133 migrations.CreateModel(
134 name="PhotoAccessStat",
135 fields=[
136 (
137 "photo",
138 models.OneToOneField(
139 on_delete=django.db.models.deletion.CASCADE,
140 primary_key=True,
141 related_name="stat",
142 serialize=False,
143 to="ivataraccount.photo",
144 ),
145 ),
146 ("access_count", models.BigIntegerField(default=0, editable=False)),
147 ],
148 options={
149 "verbose_name": "photo access stat",
150 "verbose_name_plural": "photo access stats",
151 },
152 ),
153 migrations.CreateModel(
154 name="ConfirmedEmailAccessStat",
155 fields=[
156 (
157 "email",
158 models.OneToOneField(
159 on_delete=django.db.models.deletion.CASCADE,
160 primary_key=True,
161 related_name="stat",
162 serialize=False,
163 to="ivataraccount.confirmedemail",
164 ),
165 ),
166 ("access_count", models.BigIntegerField(default=0, editable=False)),
167 ],
168 options={
169 "verbose_name": "confirmed email access stat",
170 "verbose_name_plural": "confirmed email access stats",
171 },
172 ),
173 migrations.CreateModel(
174 name="ConfirmedOpenIdAccessStat",
175 fields=[
176 (
177 "openid",
178 models.OneToOneField(
179 on_delete=django.db.models.deletion.CASCADE,
180 primary_key=True,
181 related_name="stat",
182 serialize=False,
183 to="ivataraccount.confirmedopenid",
184 ),
185 ),
186 ("access_count", models.BigIntegerField(default=0, editable=False)),
187 ],
188 options={
189 "verbose_name": "confirmed openid access stat",
190 "verbose_name_plural": "confirmed openid access stats",
191 },
192 ),
193 # 2. Copy existing access_count data from parent tables -> stat tables
194 migrations.RunPython(
195 copy_access_counts,
196 reverse_code=reverse_copy_access_counts,
197 ),
198 # 3. Drop access_count indexes via RunSQL.
199 # These were created via RunPython in migration 0021, so Django's migration
200 # state does not track them as named indexes — RemoveIndex would fail with
201 # "index not found in state". Same pattern used in migration 0023 for digest
202 # indexes. IF EXISTS makes this safe to run on databases where the indexes
203 # were never created (e.g. SQLite test databases).
204 migrations.RunSQL(
205 "DROP INDEX IF EXISTS idx_cemail_access_count;",
206 reverse_sql=migrations.RunSQL.noop,
207 ),
208 migrations.RunSQL(
209 "DROP INDEX IF EXISTS idx_cemail_user_access;",
210 reverse_sql=migrations.RunSQL.noop,
211 ),
212 migrations.RunSQL(
213 "DROP INDEX IF EXISTS idx_cemail_photo_access;",
214 reverse_sql=migrations.RunSQL.noop,
215 ),
216 migrations.RunSQL(
217 "DROP INDEX IF EXISTS idx_photo_access_count;",
218 reverse_sql=migrations.RunSQL.noop,
219 ),
220 # 4. Remove access_count columns from parent tables
221 migrations.RemoveField(
222 model_name="photo",
223 name="access_count",
224 ),
225 migrations.RemoveField(
226 model_name="confirmedemail",
227 name="access_count",
228 ),
229 migrations.RemoveField(
230 model_name="confirmedopenid",
231 name="access_count",
232 ),
233 ]