fix: update all journals in bulk (#19055)

* fix: update all journals in bulk

Issues with the previous code:

- all journal objects were loaded into memory, bloating the process for
  accounts with high volumes, only to be updated, and not used otherwise
- once loaded, iterating though each to update is a classic n+1
  performance issue, and in some cases hitting a 60 second timeout

With this code:

- perform a single SQL query to update all relevant records
- doesn't load any journal objects into memory
- since nothing is in memory, we don't need to synchronize with
  sqlalchemy session map, making this even more efficient
- no longer needs a manual flush

Fixes #19033

Signed-off-by: Mike Fiedler <miketheman@gmail.com>

* make translations

Signed-off-by: Mike Fiedler <miketheman@gmail.com>

---------

Signed-off-by: Mike Fiedler <miketheman@gmail.com>

Author: miketheman

warehouse/admin/views/users.py

@@ -16,8 +16,7 @@
 from paginate_sqlalchemy import SqlalchemyOrmPage as SQLAlchemyORMPage
 from pyramid.httpexceptions import HTTPBadRequest, HTTPMovedPermanently, HTTPSeeOther
 from pyramid.view import view_config
-from sqlalchemy import func, or_, select
-from sqlalchemy.orm import joinedload
+from sqlalchemy import func, or_, select, update
 
 from warehouse.accounts.interfaces import (
     BurnedRecoveryCode,
@@ -342,17 +341,14 @@ def _nuke_user(user, request):
 
     # Update all journals to point to `deleted-user` instead
     deleted_user = request.db.query(User).filter(User.username == "deleted-user").one()
-
-    journals = (
-        request.db.query(JournalEntry)
-        .options(joinedload(JournalEntry.submitted_by))
-        .filter(JournalEntry.submitted_by == user)
-        .all()
+    # Update in bulk to avoid loading all journal entries into memory (n+1)
+    request.db.execute(
+        update(JournalEntry)
+        .where(JournalEntry._submitted_by == user.username)
+        .values(_submitted_by=deleted_user.username)
+        .execution_options(synchronize_session=False)
     )
 
-    for journal in journals:
-        journal.submitted_by = deleted_user
-
     # Prohibit the username
     request.db.add(
         ProhibitedUserName(

warehouse/locale/messages.pot

@@ -164,7 +164,7 @@ msgstr ""
 msgid "Successful WebAuthn assertion"
 msgstr ""
 
-#: warehouse/accounts/views.py:725 warehouse/manage/views/__init__.py:862
+#: warehouse/accounts/views.py:725 warehouse/manage/views/__init__.py:855
 msgid "Recovery code accepted. The supplied code cannot be used again."
 msgstr ""
 
@@ -519,143 +519,143 @@ msgid ""
 "less."
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:316
+#: warehouse/manage/views/__init__.py:315
 msgid "Account details updated"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:347
+#: warehouse/manage/views/__init__.py:346
 #, python-brace-format
 msgid "Email ${email_address} added - check your email for a verification link"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:809
+#: warehouse/manage/views/__init__.py:802
 msgid "Recovery codes already generated"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:811
+#: warehouse/manage/views/__init__.py:804
 msgid "Generating new recovery codes will invalidate your existing codes."
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:919
+#: warehouse/manage/views/__init__.py:912
 msgid "Verify your email to create an API token."
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1021
+#: warehouse/manage/views/__init__.py:1014
 msgid "API Token does not exist."
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1053
+#: warehouse/manage/views/__init__.py:1046
 msgid "Invalid credentials. Try again"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1172
+#: warehouse/manage/views/__init__.py:1165
 msgid "Invalid alternate repository location details"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1210
+#: warehouse/manage/views/__init__.py:1203
 #, python-brace-format
 msgid "Added alternate repository '${name}'"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1243
-#: warehouse/manage/views/__init__.py:1504
-#: warehouse/manage/views/__init__.py:1589
-#: warehouse/manage/views/__init__.py:1690
-#: warehouse/manage/views/__init__.py:1790
+#: warehouse/manage/views/__init__.py:1236
+#: warehouse/manage/views/__init__.py:1497
+#: warehouse/manage/views/__init__.py:1582
+#: warehouse/manage/views/__init__.py:1683
+#: warehouse/manage/views/__init__.py:1783
 msgid "Confirm the request"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1255
+#: warehouse/manage/views/__init__.py:1248
 msgid "Invalid alternate repository id"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1266
+#: warehouse/manage/views/__init__.py:1259
 msgid "Invalid alternate repository for project"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1275
+#: warehouse/manage/views/__init__.py:1268
 #, python-brace-format
 msgid ""
 "Could not delete alternate repository - ${confirm} is not the same as "
 "${alt_repo_name}"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1305
+#: warehouse/manage/views/__init__.py:1298
 #, python-brace-format
 msgid "Deleted alternate repository '${name}'"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1373
-#: warehouse/manage/views/__init__.py:1674
-#: warehouse/manage/views/__init__.py:1782
+#: warehouse/manage/views/__init__.py:1366
+#: warehouse/manage/views/__init__.py:1667
+#: warehouse/manage/views/__init__.py:1775
 msgid ""
 "Project deletion temporarily disabled. See https://pypi.org/help#admin-"
 "intervention for details."
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1517
+#: warehouse/manage/views/__init__.py:1510
 msgid "Could not yank release - "
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1602
+#: warehouse/manage/views/__init__.py:1595
 msgid "Could not un-yank release - "
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1703
+#: warehouse/manage/views/__init__.py:1696
 msgid "Could not delete release - "
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1802
+#: warehouse/manage/views/__init__.py:1795
 msgid "Could not find file"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1807
+#: warehouse/manage/views/__init__.py:1800
 msgid "Could not delete file - "
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:1957
+#: warehouse/manage/views/__init__.py:1950
 #, python-brace-format
 msgid "Team '${team_name}' already has ${role_name} role for project"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:2064
+#: warehouse/manage/views/__init__.py:2057
 #, python-brace-format
 msgid "User '${username}' already has ${role_name} role for project"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:2131
+#: warehouse/manage/views/__init__.py:2124
 #, python-brace-format
 msgid "${username} is now ${role} of the '${project_name}' project."
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:2163
+#: warehouse/manage/views/__init__.py:2156
 #, python-brace-format
 msgid ""
 "User '${username}' does not have a verified primary email address and "
 "cannot be added as a ${role_name} for project"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:2176
+#: warehouse/manage/views/__init__.py:2169
 #: warehouse/manage/views/organizations.py:977
 #, python-brace-format
 msgid "User '${username}' already has an active invite. Please try again later."
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:2241
+#: warehouse/manage/views/__init__.py:2234
 #: warehouse/manage/views/organizations.py:1052
 #, python-brace-format
 msgid "Invitation sent to '${username}'"
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:2273
+#: warehouse/manage/views/__init__.py:2266
 msgid "Could not find role invitation."
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:2284
+#: warehouse/manage/views/__init__.py:2277
 msgid "Invitation already expired."
 msgstr ""
 
-#: warehouse/manage/views/__init__.py:2317
+#: warehouse/manage/views/__init__.py:2310
 #: warehouse/manage/views/organizations.py:1239
 #, python-brace-format
 msgid "Invitation revoked from '${username}'."

warehouse/manage/views/__init__.py

@@ -14,9 +14,8 @@
     HTTPSeeOther,
 )
 from pyramid.view import view_config, view_defaults
-from sqlalchemy import func
+from sqlalchemy import func, update
 from sqlalchemy.exc import NoResultFound
-from sqlalchemy.orm import joinedload
 from venusian import lift
 from webauthn.helpers import bytes_to_base64url
 from webob.multidict import MultiDict
@@ -467,20 +466,14 @@ def delete_account(self):
         deleted_user = (
             self.request.db.query(User).filter(User.username == "deleted-user").one()
         )
-
-        journals = (
-            self.request.db.query(JournalEntry)
-            .options(joinedload(JournalEntry.submitted_by))
-            .filter(JournalEntry.submitted_by == self.request.user)
-            .all()
+        # Update in bulk to avoid loading all journal entries into memory (n+1)
+        self.request.db.execute(
+            update(JournalEntry)
+            .where(JournalEntry._submitted_by == self.request.user.username)
+            .values(_submitted_by=deleted_user.username)
+            .execution_options(synchronize_session=False)
         )
 
-        for journal in journals:
-            journal.submitted_by = deleted_user
-
-        # Attempt to flush to identify any integrity errors before sending an email
-        self.request.db.flush()  # to identify any integrity errors
-
         # Send a notification email
         send_account_deletion_email(self.request, self.request.user)