Skip to content

Commit f132c31

Browse files
akatsoulasakatsoulas
authored
Merge pull request #6611 from akatsoulas/non-fxa-migrated-deletions
Non fxa migrated deletions
2 parents 2b79b27 + d0060f4 commit f132c31

File tree

2 files changed

+230
-0
lines changed

2 files changed

+230
-0
lines changed

‎kitsune/users/migrations/0033_batch_delete_non_migrated_users.py

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import time
2+
from datetime import timedelta
3+
from itertools import islice
4+
5+
from django.db import migrations
6+
from django.db.models import Q
7+
8+
9+
def delete_non_migrated_users(apps, schema_editor):
10+
"""
11+
Delete users where is_fxa_migrated is False and who aren't creators/owners/users of
12+
any content.
13+
"""
14+
User = apps.get_model("auth", "User")
15+
users_to_delete = User.objects.filter(profile__is_fxa_migrated=False).exclude(
16+
Q(answer_votes__isnull=False)
17+
| Q(answers__isnull=False)
18+
| Q(award_creator__isnull=False)
19+
| Q(badge__isnull=False)
20+
| Q(created_revisions__isnull=False)
21+
| Q(gallery_images__isnull=False)
22+
| Q(gallery_videos__isnull=False)
23+
| Q(inboxmessage__isnull=False)
24+
| Q(outbox__isnull=False)
25+
| Q(poll_votes__isnull=False)
26+
| Q(post__isnull=False)
27+
| Q(question_votes__isnull=False)
28+
| Q(questions__isnull=False)
29+
| Q(readied_for_l10n_revisions__isnull=False)
30+
| Q(reviewed_revisions__isnull=False)
31+
| Q(thread__isnull=False)
32+
| Q(wiki_post_set__isnull=False)
33+
| Q(wiki_thread_set__isnull=False)
34+
| Q(locales_leader__isnull=False)
35+
| Q(locales_reviewer__isnull=False)
36+
| Q(locales_editor__isnull=False)
37+
| Q(wiki_contributions__isnull=False)
38+
)
39+
40+
total_users = users_to_delete.count()
41+
if total_users == 0:
42+
print("No users to delete")
43+
return
44+
45+
print(f"Starting deletion of {total_users:,} users")
46+
start_time = time.time()
47+
deleted_count = 0
48+
batch_size = 2000
49+
50+
user_ids = users_to_delete.values_list("id", flat=True).iterator(chunk_size=batch_size)
51+
current_batch = []
52+
53+
for user_id in user_ids:
54+
current_batch.append(user_id)
55+
56+
if len(current_batch) >= batch_size:
57+
# Delete the batch using _base_manager to avoid the overridden managers
58+
# of each model through the cascade.
59+
# We don't care about the extra logic b/c the accounts that are being deleted are empty
60+
deletion_counts = User._base_manager.filter(id__in=current_batch).delete()
61+
# get the user, not the cascaded deletes
62+
user_deletes = deletion_counts[1].get("auth.User", 0)
63+
deleted_count += user_deletes
64+
current_batch = []
65+
66+
elapsed_time = time.time() - start_time
67+
avg_time_per_user = elapsed_time / deleted_count if deleted_count > 0 else 0
68+
current_rate = deleted_count / elapsed_time * 60 if elapsed_time > 0 else 0
69+
remaining_time = (
70+
(total_users - deleted_count) * avg_time_per_user if deleted_count > 0 else 0
71+
)
72+
73+
print(
74+
f"""
75+
Progress Report:
76+
---------------
77+
Users Deleted: {deleted_count:,} of {total_users:,} ({(deleted_count/total_users*100):.1f}%)
78+
Elapsed Time: {timedelta(seconds=int(elapsed_time))}
79+
Average Time per User: {avg_time_per_user:.3f} seconds
80+
Current Rate: {current_rate:.1f} users/minute
81+
Estimated Time Remaining: {timedelta(seconds=int(remaining_time))}
82+
"""
83+
)
84+
85+
if current_batch:
86+
deletion_counts = User._base_manager.filter(id__in=current_batch).delete()
87+
user_deletes = deletion_counts[1].get("auth.User", 0)
88+
deleted_count += user_deletes
89+
90+
total_time = time.time() - start_time
91+
print(
92+
f"""
93+
Deletion Complete:
94+
-----------------
95+
Total Users Deleted: {deleted_count:,} of {total_users:,}
96+
Total Time: {timedelta(seconds=int(total_time))}
97+
Average Time per User: {(total_time/deleted_count if deleted_count > 0 else 0):.3f} seconds
98+
Overall Rate: {(deleted_count/total_time*60 if total_time > 0 else 0):.1f} users/minute
99+
"""
100+
)
101+
102+
103+
def reverse_migration(apps, schema_editor):
104+
"""
105+
No reverse migration possible since deletion cannot be undone
106+
"""
107+
pass
108+
109+
110+
class Migration(migrations.Migration):
111+
dependencies = [
112+
("users", "0032_profile_account_type_alter_profile_user"),
113+
]
114+
115+
operations = [
116+
migrations.RunPython(
117+
delete_non_migrated_users,
118+
reverse_migration,
119+
),
120+
]

‎kitsune/users/tests/test_migration_0033.py

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
from django.contrib.auth.models import User
2+
from django.test import TestCase
3+
from django.db.models import Q
4+
5+
from kitsune.questions.tests import (
6+
AnswerFactory,
7+
QuestionFactory,
8+
QuestionVoteFactory,
9+
AnswerVoteFactory,
10+
)
11+
from kitsune.users.tests import ProfileFactory
12+
from kitsune.wiki.tests import RevisionFactory
13+
from kitsune.messages.models import InboxMessage, OutboxMessage
14+
from kitsune.gallery.tests import ImageFactory, VideoFactory
15+
from kitsune.forums.tests import ThreadFactory, PostFactory
16+
17+
18+
class TestDeleteNonMigratedUsersMigrationQuery(TestCase):
19+
"""Test the migration that deletes non-migrated users."""
20+
21+
def setUp(self):
22+
# Create users with different combinations of migration status and content creation
23+
24+
# Case 1: Non-migrated user with no content (should be deleted)
25+
ProfileFactory(user__username="user1", is_fxa_migrated=False)
26+
27+
# Case 2: Non-migrated user with a question (should be kept)
28+
user2 = ProfileFactory(user__username="user2", is_fxa_migrated=False).user
29+
QuestionFactory(creator=user2)
30+
31+
# Case 3: Non-migrated user with an answer (should be kept)
32+
user3 = ProfileFactory(user__username="user3", is_fxa_migrated=False).user
33+
AnswerFactory(creator=user3)
34+
35+
# Case 4: Non-migrated user with a revision (should be kept)
36+
user4 = ProfileFactory(user__username="user4", is_fxa_migrated=False).user
37+
RevisionFactory(creator=user4)
38+
39+
# Case 5: Migrated user with no content (should be kept)
40+
ProfileFactory(user__username="user5", is_fxa_migrated=True)
41+
42+
# Case 6: Non-migrated user with a question vote (should be kept)
43+
user6 = ProfileFactory(user__username="user6", is_fxa_migrated=False).user
44+
QuestionVoteFactory(creator=user6)
45+
46+
# Case 7: Non-migrated user with an answer vote (should be kept)
47+
user7 = ProfileFactory(user__username="user7", is_fxa_migrated=False).user
48+
AnswerVoteFactory(creator=user7)
49+
50+
# Case 8: Non-migrated user who is a sender of inbox messages (should be kept)
51+
user8 = ProfileFactory(user__username="user8", is_fxa_migrated=False).user
52+
InboxMessage.objects.create(to=user2, sender=user8, message="test")
53+
54+
# Case 9: Non-migrated user with an outbox message (should be kept)
55+
user9 = ProfileFactory(user__username="user9", is_fxa_migrated=False).user
56+
outbox_msg = OutboxMessage.objects.create(sender=user9, message="test")
57+
outbox_msg.to.add(user2)
58+
59+
# Case 10: Non-migrated user who is a sender of inbox messages (should be kept)
60+
user10 = ProfileFactory(user__username="user10", is_fxa_migrated=False).user
61+
InboxMessage.objects.create(to=user2, sender=user10, message="test")
62+
63+
# Case 11: Non-migrated user with a gallery image (should be kept)
64+
user11 = ProfileFactory(user__username="user11", is_fxa_migrated=False).user
65+
ImageFactory(creator=user11)
66+
67+
# Case 12: Non-migrated user with a gallery video (should be kept)
68+
user12 = ProfileFactory(user__username="user12", is_fxa_migrated=False).user
69+
VideoFactory(creator=user12)
70+
71+
# Case 13: Non-migrated user with a forum thread (should be kept)
72+
user13 = ProfileFactory(user__username="user13", is_fxa_migrated=False).user
73+
ThreadFactory(creator=user13)
74+
75+
# Case 14: Non-migrated user with a forum post (should be kept)
76+
user14 = ProfileFactory(user__username="user14", is_fxa_migrated=False).user
77+
thread = ThreadFactory()
78+
PostFactory(thread=thread, author=user14)
79+
80+
# Case 15: Non-migrated user who reviewed a revision (should be kept)
81+
user15 = ProfileFactory(user__username="user15", is_fxa_migrated=False).user
82+
RevisionFactory(reviewer=user15)
83+
84+
def test_direct_query_logic(self):
85+
"""Test the query logic of the migration directly without going through apps."""
86+
# Query using the same logic as the migration but with direct model references
87+
users_to_delete = User.objects.filter(profile__is_fxa_migrated=False).exclude(
88+
Q(answer_votes__isnull=False)
89+
| Q(answers__isnull=False)
90+
| Q(award_creator__isnull=False)
91+
| Q(badge__isnull=False)
92+
| Q(created_revisions__isnull=False)
93+
| Q(gallery_images__isnull=False)
94+
| Q(gallery_videos__isnull=False)
95+
| Q(inboxmessage__isnull=False)
96+
| Q(outbox__isnull=False)
97+
| Q(poll_votes__isnull=False)
98+
| Q(post__isnull=False)
99+
| Q(question_votes__isnull=False)
100+
| Q(questions__isnull=False)
101+
| Q(readied_for_l10n_revisions__isnull=False)
102+
| Q(reviewed_revisions__isnull=False)
103+
| Q(thread__isnull=False)
104+
| Q(wiki_post_set__isnull=False)
105+
| Q(wiki_thread_set__isnull=False)
106+
)
107+
108+
# Only user1 should be in this queryset
109+
self.assertEqual(users_to_delete.count(), 1)
110+
self.assertEqual(users_to_delete.first().username, "user1")

0 commit comments

Comments
 (0)