Coverage for ivatar/photo_cache.py: 85%

1""" 

2Redis and in-memory caching for resized user photo payloads with model invalidation. 

3""" 

4 

5import contextlib 

6import logging 

7import threading 

8from io import BytesIO 

9from typing import Dict, List, Optional, Sequence, Set, Tuple, Union 

10 

11from django.conf import settings 

12from django.core.cache import cache, caches 

13 

14logger = logging.getLogger("ivatar") 

15 

16# Thread-safe in-memory L1 cache for resized photo bytes: (photo_id, size, format) -> bytes 

17_user_photos_cache: Dict[Tuple[int, int, str], bytes] = {} 

18_user_photos_cache_lock = threading.Lock() 

19_MAX_USER_PHOTO_CACHE_SIZE = getattr(settings, "MAX_USER_PHOTO_CACHE_SIZE", 500) 

20 

21# Common sizes used across Libravatar / Ivatar 

22COMMON_SIZES = (28, 48, 80, 128, 256, 512) 

23DEFAULT_PREGENERATE_SIZES = (28, 48, 80, 128, 256) 

24COMMON_FORMATS = ("png", "jpg", "jpeg", "gif", "webp") 

25 

26 

27def _get_cache_backend(): 

28 """ 

29 Get the configured cache backend for user photos, falling back to default. 

30 """ 

31 cache_alias = getattr(settings, "PHOTO_CACHE_ALIAS", "default") 

32 try: 

33 return caches[cache_alias] 

34 except Exception: # pylint: disable=broad-exception-caught 

35 return cache 

36 

37 

38def get_cached_photo(photo_id: int, size: int, format_type: str) -> Optional[BytesIO]: 

39 """ 

40 Retrieve cached resized photo bytes. 

41 First checks the local process memory cache (L1), then falls back to Redis/Django cache (L2). 

42 Returns a BytesIO stream if found, or None. 

43 """ 

44 if not photo_id: 

45 return None 

46 

47 cache_key = (photo_id, size, format_type) 

48 

49 # 1. Check L1 in-process memory cache 

50 with _user_photos_cache_lock: 

51 if cache_key in _user_photos_cache: 

52 return BytesIO(_user_photos_cache[cache_key]) 

53 

54 # 2. Check L2 Redis / Django cache 

55 redis_key = f"photo_cache:{photo_id}:{size}:{format_type}" 

56 cache_backend = _get_cache_backend() 

57 max_cache_size = getattr(settings, "MAX_USER_PHOTO_CACHE_SIZE", 500) 

58 with contextlib.suppress(Exception): 

59 data_bytes = cache_backend.get(redis_key) 

60 if data_bytes is not None: 

61 # Populate L1 in-process cache 

62 with _user_photos_cache_lock: 

63 if len(_user_photos_cache) >= max_cache_size: 

64 remove_keys = list(_user_photos_cache.keys())[:50] 

65 for rk in remove_keys: 

66 del _user_photos_cache[rk] 

67 _user_photos_cache[cache_key] = data_bytes 

68 return BytesIO(data_bytes) 

69 

70 return None 

71 

72 

73def set_cached_photo( 

74 photo_id: int, size: int, format_type: str, photo_bytes: bytes 

75) -> None: 

76 """ 

77 Store resized photo bytes into L1 memory cache and L2 Redis / Django cache. 

78 Also tracks cached (size, format) keys for fast and complete invalidation. 

79 """ 

80 if not photo_id or not photo_bytes: 

81 return 

82 

83 cache_key = (photo_id, size, format_type) 

84 max_cache_size = getattr(settings, "MAX_USER_PHOTO_CACHE_SIZE", 500) 

85 

86 # 1. Save in L1 in-process memory cache 

87 with _user_photos_cache_lock: 

88 if len(_user_photos_cache) >= max_cache_size: 

89 remove_keys = list(_user_photos_cache.keys())[:50] 

90 for rk in remove_keys: 

91 del _user_photos_cache[rk] 

92 _user_photos_cache[cache_key] = photo_bytes 

93 

94 # 2. Save in L2 Redis / Django cache 

95 redis_key = f"photo_cache:{photo_id}:{size}:{format_type}" 

96 tracking_key = f"photo_cache_keys:{photo_id}" 

97 timeout = getattr(settings, "PHOTO_CACHE_TIMEOUT", 604800) # 7 days default 

98 cache_backend = _get_cache_backend() 

99 

100 with contextlib.suppress(Exception): 

101 cache_backend.set(redis_key, photo_bytes, timeout=timeout) 

102 # Update tracking list of cached keys for this photo 

103 tracked_keys: Set[str] = set(cache_backend.get(tracking_key) or []) 

104 tracked_keys.add(redis_key) 

105 cache_backend.set(tracking_key, list(tracked_keys), timeout=timeout) 

106 

107 

108def invalidate_photo_cache(photo_id: int) -> None: 

109 """ 

110 Invalidate all cached resized variants of a photo across L1 and L2 caches. 

111 Called on Photo.save() and Photo.delete() and post_save/post_delete signals. 

112 """ 

113 if not photo_id: 

114 return 

115 

116 # 1. Clear from L1 in-process memory cache 

117 with _user_photos_cache_lock: 

118 keys_to_remove = [k for k in _user_photos_cache if k[0] == photo_id] 

119 for k in keys_to_remove: 

120 del _user_photos_cache[k] 

121 

122 # 2. Clear from L2 Redis / Django cache 

123 cache_backend = _get_cache_backend() 

124 tracking_key = f"photo_cache_keys:{photo_id}" 

125 

126 with contextlib.suppress(Exception): 

127 keys_to_delete: Set[str] = set() 

128 

129 # Check tracked keys 

130 tracked = cache_backend.get(tracking_key) 

131 if tracked: 

132 keys_to_delete.update(tracked) 

133 

134 # Also purge common sizes across common formats as a safety net 

135 for s in COMMON_SIZES: 

136 for fmt in COMMON_FORMATS: 

137 keys_to_delete.add(f"photo_cache:{photo_id}:{s}:{fmt}") 

138 

139 if hasattr(cache_backend, "delete_many"): 

140 cache_backend.delete_many(list(keys_to_delete)) 

141 else: 

142 for k in keys_to_delete: 

143 cache_backend.delete(k) 

144 

145 cache_backend.delete(tracking_key) 

146 

147 # If cache backend supports pattern deletion (e.g. django-redis) 

148 if hasattr(cache_backend, "delete_pattern"): 

149 cache_backend.delete_pattern(f"photo_cache:{photo_id}:*") 

150 

151 

152def clear_local_photo_cache() -> None: 

153 """ 

154 Clear in-process memory cache (primarily for tests). 

155 """ 

156 with _user_photos_cache_lock: 

157 _user_photos_cache.clear() 

158 

159 

160def render_photo_thumbnail( 

161 photo_data: bytes, imgformat: str, size: int, target_format: Optional[str] = None 

162) -> bytes: 

163 """ 

164 Render and return binary bytes for a photo resized to (size, size). 

165 Supports animated GIFs, dynamic resampling filters (e.g. BOX/BILINEAR/LANCZOS), 

166 WebP content negotiation, and image quality settings. 

167 """ 

168 from PIL import Image 

169 from ivatar.utils import get_resample_filter, resize_animated_gif 

170 from ivatar.ivataraccount.models import pil_format 

171 

172 photodata = Image.open(BytesIO(photo_data)) 

173 data = BytesIO() 

174 

175 output_format = target_format if target_format is not None else imgformat 

176 

177 # Animated GIFs need additional handling 

178 if imgformat == "gif" and getattr(photodata, "is_animated", False): 

179 data = resize_animated_gif(photodata, (size, size)) 

180 else: 

181 # If the image is smaller than what was requested, we need to use resize 

182 resample_filter = get_resample_filter(size, photodata.size) 

183 if photodata.size[0] < size or photodata.size[1] < size: 

184 photodata = photodata.resize((size, size), resample_filter) 

185 else: 

186 photodata.thumbnail((size, size), resample_filter) 

187 

188 p_format = pil_format(output_format) 

189 if p_format == "JPEG" and photodata.mode in ("RGBA", "LA", "P"): 

190 photodata = photodata.convert("RGB") 

191 elif p_format == "WEBP" and photodata.mode in ("P", "1", "CMYK"): 

192 photodata = photodata.convert("RGBA") 

193 

194 if p_format == "WEBP": 

195 webp_quality = getattr(settings, "WEBP_QUALITY", 85) 

196 photodata.save(data, "WEBP", quality=webp_quality, method=4) 

197 elif p_format == "JPEG": 

198 jpeg_quality = getattr(settings, "JPEG_QUALITY", 85) 

199 photodata.save(data, p_format, quality=jpeg_quality) 

200 else: 

201 photodata.save(data, p_format) 

202 

203 return data.getvalue() 

204 

205 

206def _do_pregenerate( 

207 photo_id: int, photo_data: bytes, imgformat: str, target_sizes: Sequence[int] 

208) -> List[int]: 

209 """ 

210 Execute thumbnail generation and caching for the specified target sizes. 

211 Pre-caches native format and WebP format (if enabled). 

212 """ 

213 generated: List[int] = [] 

214 normalized_format = "jpeg" if imgformat == "jpg" else imgformat 

215 pregen_webp = getattr(settings, "PREGENERATE_WEBP_THUMBNAILS", True) 

216 

217 for size in target_sizes: 

218 try: 

219 # 1. Native format thumbnail 

220 thumb_bytes = render_photo_thumbnail(photo_data, imgformat, size) 

221 set_cached_photo(photo_id, size, normalized_format, thumb_bytes) 

222 

223 # 2. WebP format thumbnail 

224 if pregen_webp and normalized_format != "webp": 

225 webp_bytes = render_photo_thumbnail( 

226 photo_data, imgformat, size, target_format="webp" 

227 ) 

228 set_cached_photo(photo_id, size, "webp", webp_bytes) 

229 

230 generated.append(size) 

231 except Exception as exc: # pylint: disable=broad-exception-caught 

232 logger.warning( 

233 "Failed to pre-generate thumbnail for photo %s at size %s: %s", 

234 photo_id, 

235 size, 

236 exc, 

237 ) 

238 logger.debug( 

239 "Pre-generated %d thumbnails for photo %s: sizes=%s", 

240 len(generated), 

241 photo_id, 

242 generated, 

243 ) 

244 return generated 

245 

246 

247def pregenerate_photo_thumbnails( 

248 photo_or_id, 

249 photo_data: Optional[bytes] = None, 

250 imgformat: Optional[str] = None, 

251 sizes: Optional[Sequence[int]] = None, 

252 async_generation: Optional[bool] = None, 

253) -> List[int]: 

254 """ 

255 Pre-generate and cache standard thumbnail sizes for a photo. 

256 Target sizes default to PREGENERATE_PHOTO_SIZES (28, 48, 80, 128, 256). 

257 If async_generation is True, executes in a background daemon thread. 

258 Returns the list of successfully generated sizes. 

259 """ 

260 photo_id = None 

261 if hasattr(photo_or_id, "id") and hasattr(photo_or_id, "data"): 

262 photo_id = photo_or_id.id 

263 if photo_data is None and photo_or_id.data: 

264 photo_data = bytes(photo_or_id.data) 

265 if imgformat is None: 

266 imgformat = getattr(photo_or_id, "format", None) 

267 elif photo_or_id is not None: 

268 with contextlib.suppress(Exception): 

269 photo_id = int(photo_or_id) 

270 

271 if not photo_id: 

272 return [] 

273 

274 if photo_data is None or imgformat is None: 

275 try: 

276 from ivatar.ivataraccount.models import Photo 

277 photo = Photo.objects.get(pk=photo_id) 

278 if photo_data is None and photo.data: 

279 photo_data = bytes(photo.data) 

280 if imgformat is None: 

281 imgformat = photo.format 

282 except Exception as exc: # pylint: disable=broad-exception-caught 

283 logger.warning( 

284 "Could not load photo %s for thumbnail pre-generation: %s", 

285 photo_id, 

286 exc, 

287 ) 

288 return [] 

289 

290 if not photo_data or not imgformat: 

291 return [] 

292 

293 target_sizes = sizes or getattr( 

294 settings, "PREGENERATE_PHOTO_SIZES", DEFAULT_PREGENERATE_SIZES 

295 ) 

296 

297 if async_generation is None: 

298 async_generation = getattr(settings, "PREGENERATE_THUMBNAILS_ASYNC", False) 

299 

300 if async_generation: 

301 thread = threading.Thread( 

302 target=_do_pregenerate, 

303 args=(photo_id, photo_data, imgformat, target_sizes), 

304 daemon=True, 

305 ) 

306 thread.start() 

307 return list(target_sizes) 

308 

309 return _do_pregenerate(photo_id, photo_data, imgformat, target_sizes) 

310 

311 

312def invalidate_digest_cache( # pylint: disable=too-many-locals,too-many-branches,too-many-statements 

313 digests: Union[str, Sequence[str]] 

314) -> None: 

315 """ 

316 Invalidate all cached avatar responses, default avatar renders, 

317 Gravatar proxy/test entries, and related cache entries for the given digest(s). 

318 Supports single digest string or sequence of digests. 

319 """ 

320 if isinstance(digests, str): 

321 digests = [digests] 

322 digest_set = {d.strip().lower() for d in digests if d and isinstance(d, str)} 

323 if not digest_set: 

324 return 

325 

326 # 1. Clear in-process default avatar cache in ivatar.views 

327 try: 

328 from ivatar.views import ( # pylint: disable=import-outside-toplevel 

329 _default_avatars_png_cache, 

330 _default_avatars_png_cache_lock, 

331 ) 

332 

333 with _default_avatars_png_cache_lock: 

334 keys_to_del = [k for k in _default_avatars_png_cache if k[1] in digest_set] 

335 for k in keys_to_del: 

336 del _default_avatars_png_cache[k] 

337 except Exception as exc: # pylint: disable=broad-exception-caught 

338 logger.debug("Could not clear in-memory default avatar cache: %s", exc) 

339 

340 # 2. Collect unique cache backends 

341 backends = [] 

342 aliases = ["default"] 

343 for alias_attr in ("AVATAR_CACHE_ALIAS", "PHOTO_CACHE_ALIAS"): 

344 if hasattr(settings, alias_attr): 

345 aliases.append(getattr(settings, alias_attr)) 

346 if hasattr(settings, "CACHES") and "filesystem" in settings.CACHES: 

347 aliases.append("filesystem") 

348 

349 seen_backend_ids = set() 

350 for alias in aliases: 

351 try: 

352 b = caches[alias] 

353 if id(b) not in seen_backend_ids: 

354 seen_backend_ids.add(id(b)) 

355 backends.append(b) 

356 except Exception: # pylint: disable=broad-exception-caught 

357 pass 

358 

359 # 3. Build candidate keys to delete for backends that do not support delete_pattern 

360 common_sizes = (0, 28, 48, 80, 128, 256, 512) 

361 common_styles = ( 

362 "retro", 

363 "monsterid", 

364 "wavatar", 

365 "identicon", 

366 "robohash", 

367 "pagan", 

368 "mmng", 

369 ) 

370 common_hosts = [ 

371 "https://seccdn.libravatar.org", 

372 "https://cdn.libravatar.org", 

373 "http://testserver", 

374 ] 

375 if getattr(settings, "SECURE_BASE_URL", None): 

376 base = settings.SECURE_BASE_URL.rstrip("/") 

377 if base.endswith("/avatar"): 

378 base = base[:-7] 

379 common_hosts.append(base) 

380 if getattr(settings, "BASE_URL", None): 

381 base = settings.BASE_URL.rstrip("/") 

382 if base.endswith("/avatar"): 

383 base = base[:-7] 

384 common_hosts.append(base) 

385 

386 hosts = list(set(h.rstrip("/") for h in common_hosts if h)) 

387 

388 keys_to_delete: Set[str] = set() 

389 

390 for digest in digest_set: 

391 # Direct digest / URI suffix keys 

392 keys_to_delete.add(f"{digest}") 

393 keys_to_delete.add(f"{digest}:standard") 

394 keys_to_delete.add(f"{digest}:webp") 

395 

396 for host in hosts: 

397 for path_prefix in (f"/avatar/{digest}", f"/avatar/{digest}/"): 

398 base_uri = f"{host}{path_prefix}" 

399 keys_to_delete.add(base_uri) 

400 keys_to_delete.add(f"{base_uri}:standard") 

401 keys_to_delete.add(f"{base_uri}:webp") 

402 

403 for s in common_sizes: 

404 for q in ( 

405 f"?s={s}", 

406 f"?size={s}", 

407 f"?s={s}&default=mm", 

408 f"?s={s}&d=mm", 

409 f"?s={s}&forcedefault=y", 

410 f"?s={s}&d=404", 

411 ): 

412 uri_with_q = f"{base_uri}{q}" 

413 keys_to_delete.add(uri_with_q) 

414 keys_to_delete.add(f"{uri_with_q}:standard") 

415 keys_to_delete.add(f"{uri_with_q}:webp") 

416 

417 # Gravatarproxy URLs 

418 keys_to_delete.add(f"{host}/gravatarproxy/{digest}") 

419 for s in common_sizes: 

420 keys_to_delete.add(f"{host}/gravatarproxy/{digest}?s={s}") 

421 keys_to_delete.add(f"{host}/gravatarproxy/{digest}?s={s}&default=mm") 

422 

423 # Default avatar cache keys 

424 for style in common_styles: 

425 for s in common_sizes: 

426 keys_to_delete.add(f"default_avatar:{style}:{digest}:{s}") 

427 

428 # Gravatar proxy/error/test keys 

429 for s in common_sizes: 

430 keys_to_delete.add(f"gravatar_proxy_img:{digest}:{s}") 

431 keys_to_delete.add(f"gravatar_proxy_err:{digest}:{s}") 

432 keys_to_delete.add(f"gravatar_test:{digest}:{s}") 

433 

434 key_list = list(keys_to_delete) 

435 

436 # 4. Invalidate on each cache backend 

437 for backend in backends: 

438 with contextlib.suppress(Exception): 

439 # If backend supports pattern deletion (e.g. django-redis) 

440 if hasattr(backend, "delete_pattern"): 

441 for digest in digest_set: 

442 backend.delete_pattern(f"*{digest}*") 

443 elif hasattr(backend, "delete_many"): 

444 backend.delete_many(key_list) 

445 else: 

446 for k in key_list: 

447 backend.delete(k)