Fix bulk_import.py stalling on real-scale exports (1M+ ids)

asyncio.gather(*(_worker(id) for id in ids)) eagerly wraps every
argument into a Task before awaiting any of them -- with a semaphore
bounding concurrent *execution* but not concurrent Task *creation*,
running this against the real movie export (1,240,787 ids) meant
constructing 1.24 million live Task + coroutine objects up front,
which looked indistinguishable from hanging for minutes before any
work could start, confirmed live in production tonight.

Replaced with a fixed pool of _CONCURRENCY (50) consumer tasks pulling
from a plain asyncio.Queue pre-loaded with every id -- only 50 Task
objects ever exist regardless of catalog size. Verified: 500K enqueues
against no-op stubs now complete in 0.59s (previously: the setup loop
alone dominated), and the real bottleneck is correctly back to the
per-id DB round-trip + Redis push, gated by the pool of 50 -- not
Python-level Task scheduling overhead.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

app/scripts/bulk_import.py

@@ -87,25 +87,41 @@ async def _bulk_enqueue(entity_type: str, *, dry_run: bool, include_seasons: boo
         return
 
     queue = await get_arq_pool()
-    semaphore = asyncio.Semaphore(_CONCURRENCY)
+    # A fixed pool of _CONCURRENCY consumer tasks pulling from a plain
+    # asyncio.Queue -- NOT asyncio.gather() over one Task per id. gather()
+    # eagerly wraps every argument into a Task before awaiting any of them,
+    # so with a catalog in the millions that means millions of live Task +
+    # coroutine objects created up front regardless of how tightly a
+    # semaphore bounds actual concurrent execution -- a real bottleneck
+    # (and memory risk) discovered running this against the real 1.24M-
+    # movie export, where it looked hung for minutes just getting through
+    # that setup before any work could start.
+    id_queue: asyncio.Queue[str] = asyncio.Queue()
+    for external_id in ids:
+        id_queue.put_nowait(external_id)
+
     enqueued = 0
     lock = asyncio.Lock()
 
-    async def _worker(external_id: str) -> None:
+    async def _consumer() -> None:
         nonlocal enqueued
-        async with semaphore:
+        while True:
+            try:
+                external_id = id_queue.get_nowait()
+            except asyncio.QueueEmpty:
+                return
             await _enqueue_one(queue, entity_type, external_id, include_seasons=include_seasons)
-        async with lock:
-            enqueued += 1
-            if enqueued % 5000 == 0:
-                logger.info(
-                    "bulk_import_progress",
-                    entity_type=entity_type,
-                    enqueued=enqueued,
-                    total=len(ids),
-                )
+            async with lock:
+                enqueued += 1
+                if enqueued % 5000 == 0:
+                    logger.info(
+                        "bulk_import_progress",
+                        entity_type=entity_type,
+                        enqueued=enqueued,
+                        total=len(ids),
+                    )
 
-    await asyncio.gather(*(_worker(external_id) for external_id in ids))
+    await asyncio.gather(*(_consumer() for _ in range(_CONCURRENCY)))
     logger.info("bulk_import_complete", entity_type=entity_type, enqueued=enqueued)