From 2463b53363a621c1e18223bda5c47a663707a22c Mon Sep 17 00:00:00 2001 From: Matt Jankowski Date: Thu, 21 Dec 2023 03:51:03 -0500 Subject: [PATCH] More duplicates in cli maintenance spec, misc bug fixes (#28449) --- lib/mastodon/cli/maintenance.rb | 40 +- spec/lib/mastodon/cli/maintenance_spec.rb | 426 +++++++++++++++++++++- 2 files changed, 443 insertions(+), 23 deletions(-) diff --git a/lib/mastodon/cli/maintenance.rb b/lib/mastodon/cli/maintenance.rb index 98067c6e3..7b3a9852a 100644 --- a/lib/mastodon/cli/maintenance.rb +++ b/lib/mastodon/cli/maintenance.rb @@ -40,6 +40,10 @@ module Mastodon::CLI class BulkImport < ApplicationRecord; end class SoftwareUpdate < ApplicationRecord; end + class DomainBlock < ApplicationRecord + scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), domain')) } + end + class PreviewCard < ApplicationRecord self.inheritance_column = false end @@ -249,19 +253,7 @@ module Mastodon::CLI say 'Deduplicating user records…' - # Deduplicating email - ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row| - users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse - ref_user = users.shift - say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow - say "e-mail will be disabled for the following accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow - say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow - - users.each_with_index do |user, index| - user.update!(email: "#{index} " + user.email) - end - end - + deduplicate_users_process_email deduplicate_users_process_confirmation_token deduplicate_users_process_remember_token deduplicate_users_process_password_token @@ -280,6 +272,20 @@ module Mastodon::CLI ActiveRecord::Base.connection.execute('REINDEX INDEX index_users_on_unconfirmed_email;') if migrator_version >= 2023_07_02_151753 end + def deduplicate_users_process_email + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users GROUP BY email HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse + ref_user = users.shift + say "Multiple users registered with e-mail address #{ref_user.email}.", :yellow + say "e-mail will be disabled for the following accounts: #{users.map { |user| user.account.acct }.join(', ')}", :yellow + say 'Please reach out to them and set another address with `tootctl account modify` or delete them.', :yellow + + users.each_with_index do |user, index| + user.update!(email: "#{index} " + user.email) + end + end + end + def deduplicate_users_process_confirmation_token ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE confirmation_token IS NOT NULL GROUP BY confirmation_token HAVING count(*) > 1").each do |row| users = User.where(id: row['ids'].split(',')).sort_by(&:created_at).reverse.drop(1) @@ -571,7 +577,7 @@ module Mastodon::CLI say 'Deduplicating webhooks…' ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM webhooks GROUP BY url HAVING count(*) > 1").each do |row| - Webhooks.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) + Webhook.where(id: row['ids'].split(',')).sort_by(&:id).reverse.drop(1).each(&:destroy) end say 'Restoring webhooks indexes…' @@ -604,11 +610,7 @@ module Mastodon::CLI say 'Please chose the one to keep unchanged, other ones will be automatically renamed.' - ref_id = ask('Account to keep unchanged:') do |q| - q.required true - q.default 0 - q.convert :int - end + ref_id = ask('Account to keep unchanged:', required: true, default: 0).to_i accounts.delete_at(ref_id) diff --git a/spec/lib/mastodon/cli/maintenance_spec.rb b/spec/lib/mastodon/cli/maintenance_spec.rb index 353bf08b6..ca492bbf6 100644 --- a/spec/lib/mastodon/cli/maintenance_spec.rb +++ b/spec/lib/mastodon/cli/maintenance_spec.rb @@ -62,6 +62,7 @@ describe Mastodon::CLI::Maintenance do context 'with duplicate accounts' do before do prepare_duplicate_data + choose_local_account_to_keep end let(:duplicate_account_username) { 'username' } @@ -71,21 +72,37 @@ describe Mastodon::CLI::Maintenance do expect { subject } .to output_results( 'Deduplicating accounts', + 'Multiple local accounts were found for', 'Restoring index_accounts_on_username_and_domain_lower', 'Reindexing textual indexes on accounts…', 'Finished!' ) - .and change(duplicate_accounts, :count).from(2).to(1) + .and change(duplicate_remote_accounts, :count).from(2).to(1) + .and change(duplicate_local_accounts, :count).from(2).to(1) end - def duplicate_accounts + def duplicate_remote_accounts Account.where(username: duplicate_account_username, domain: duplicate_account_domain) end + def duplicate_local_accounts + Account.where(username: duplicate_account_username, domain: nil) + end + def prepare_duplicate_data ActiveRecord::Base.connection.remove_index :accounts, name: :index_accounts_on_username_and_domain_lower - Fabricate(:account, username: duplicate_account_username, domain: duplicate_account_domain) - Fabricate.build(:account, username: duplicate_account_username, domain: duplicate_account_domain).save(validate: false) + _remote_account = Fabricate(:account, username: duplicate_account_username, domain: duplicate_account_domain) + _remote_account_dupe = Fabricate.build(:account, username: duplicate_account_username, domain: duplicate_account_domain).save(validate: false) + _local_account = Fabricate(:account, username: duplicate_account_username, domain: nil) + _local_account_dupe = Fabricate.build(:account, username: duplicate_account_username, domain: nil).save(validate: false) + end + + def choose_local_account_to_keep + allow(cli.shell) + .to receive(:ask) + .with(/Account to keep unchanged/, anything) + .and_return('0') + .once end end @@ -175,6 +192,407 @@ describe Mastodon::CLI::Maintenance do end end + context 'with duplicate account_domain_blocks' do + before do + prepare_duplicate_data + end + + let(:duplicate_domain) { 'example.host' } + let(:account) { Fabricate(:account) } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Removing duplicate account domain blocks', + 'Restoring account domain blocks indexes', + 'Finished!' + ) + .and change(duplicate_account_domain_blocks, :count).from(2).to(1) + end + + def duplicate_account_domain_blocks + AccountDomainBlock.where(account: account, domain: duplicate_domain) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :account_domain_blocks, [:account_id, :domain] + Fabricate(:account_domain_block, account: account, domain: duplicate_domain) + Fabricate.build(:account_domain_block, account: account, domain: duplicate_domain).save(validate: false) + end + end + + context 'with duplicate announcement_reactions' do + before do + prepare_duplicate_data + end + + let(:account) { Fabricate(:account) } + let(:announcement) { Fabricate(:announcement) } + let(:name) { Fabricate(:custom_emoji).shortcode } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Removing duplicate announcement reactions', + 'Restoring announcement_reactions indexes', + 'Finished!' + ) + .and change(duplicate_announcement_reactions, :count).from(2).to(1) + end + + def duplicate_announcement_reactions + AnnouncementReaction.where(account: account, announcement: announcement, name: name) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :announcement_reactions, [:account_id, :announcement_id, :name] + Fabricate(:announcement_reaction, account: account, announcement: announcement, name: name) + Fabricate.build(:announcement_reaction, account: account, announcement: announcement, name: name).save(validate: false) + end + end + + context 'with duplicate conversations' do + before do + prepare_duplicate_data + end + + let(:uri) { 'https://example.host/path' } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating conversations', + 'Restoring conversations indexes', + 'Finished!' + ) + .and change(duplicate_conversations, :count).from(2).to(1) + end + + def duplicate_conversations + Conversation.where(uri: uri) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :conversations, :uri + Fabricate(:conversation, uri: uri) + Fabricate.build(:conversation, uri: uri).save(validate: false) + end + end + + context 'with duplicate custom_emojis' do + before do + prepare_duplicate_data + end + + let(:duplicate_shortcode) { 'wowzers' } + let(:duplicate_domain) { 'example.host' } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating custom_emojis', + 'Restoring custom_emojis indexes', + 'Finished!' + ) + .and change(duplicate_custom_emojis, :count).from(2).to(1) + end + + def duplicate_custom_emojis + CustomEmoji.where(shortcode: duplicate_shortcode, domain: duplicate_domain) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :custom_emojis, [:shortcode, :domain] + Fabricate(:custom_emoji, shortcode: duplicate_shortcode, domain: duplicate_domain) + Fabricate.build(:custom_emoji, shortcode: duplicate_shortcode, domain: duplicate_domain).save(validate: false) + end + end + + context 'with duplicate custom_emoji_categories' do + before do + prepare_duplicate_data + end + + let(:duplicate_name) { 'name_value' } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating custom_emoji_categories', + 'Restoring custom_emoji_categories indexes', + 'Finished!' + ) + .and change(duplicate_custom_emoji_categories, :count).from(2).to(1) + end + + def duplicate_custom_emoji_categories + CustomEmojiCategory.where(name: duplicate_name) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :custom_emoji_categories, :name + Fabricate(:custom_emoji_category, name: duplicate_name) + Fabricate.build(:custom_emoji_category, name: duplicate_name).save(validate: false) + end + end + + context 'with duplicate domain_allows' do + before do + prepare_duplicate_data + end + + let(:domain) { 'example.host' } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating domain_allows', + 'Restoring domain_allows indexes', + 'Finished!' + ) + .and change(duplicate_domain_allows, :count).from(2).to(1) + end + + def duplicate_domain_allows + DomainAllow.where(domain: domain) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :domain_allows, :domain + Fabricate(:domain_allow, domain: domain) + Fabricate.build(:domain_allow, domain: domain).save(validate: false) + end + end + + context 'with duplicate domain_blocks' do + before do + prepare_duplicate_data + end + + let(:domain) { 'example.host' } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating domain_blocks', + 'Restoring domain_blocks indexes', + 'Finished!' + ) + .and change(duplicate_domain_blocks, :count).from(2).to(1) + end + + def duplicate_domain_blocks + DomainBlock.where(domain: domain) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :domain_blocks, :domain + Fabricate(:domain_block, domain: domain) + Fabricate.build(:domain_block, domain: domain).save(validate: false) + end + end + + context 'with duplicate email_domain_blocks' do + before do + prepare_duplicate_data + end + + let(:domain) { 'example.host' } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating email_domain_blocks', + 'Restoring email_domain_blocks indexes', + 'Finished!' + ) + .and change(duplicate_email_domain_blocks, :count).from(2).to(1) + end + + def duplicate_email_domain_blocks + EmailDomainBlock.where(domain: domain) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :email_domain_blocks, :domain + Fabricate(:email_domain_block, domain: domain) + Fabricate.build(:email_domain_block, domain: domain).save(validate: false) + end + end + + context 'with duplicate media_attachments' do + before do + prepare_duplicate_data + end + + let(:shortcode) { 'codenam' } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating media_attachments', + 'Restoring media_attachments indexes', + 'Finished!' + ) + .and change(duplicate_media_attachments, :count).from(2).to(1) + end + + def duplicate_media_attachments + MediaAttachment.where(shortcode: shortcode) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :media_attachments, :shortcode + Fabricate(:media_attachment, shortcode: shortcode) + Fabricate.build(:media_attachment, shortcode: shortcode).save(validate: false) + end + end + + context 'with duplicate preview_cards' do + before do + prepare_duplicate_data + end + + let(:url) { 'https://example.host/path' } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating preview_cards', + 'Restoring preview_cards indexes', + 'Finished!' + ) + .and change(duplicate_preview_cards, :count).from(2).to(1) + end + + def duplicate_preview_cards + PreviewCard.where(url: url) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :preview_cards, :url + Fabricate(:preview_card, url: url) + Fabricate.build(:preview_card, url: url).save(validate: false) + end + end + + context 'with duplicate statuses' do + before do + prepare_duplicate_data + end + + let(:uri) { 'https://example.host/path' } + let(:account) { Fabricate(:account) } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating statuses', + 'Restoring statuses indexes', + 'Finished!' + ) + .and change(duplicate_statuses, :count).from(2).to(1) + end + + def duplicate_statuses + Status.where(uri: uri) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :statuses, :uri + Fabricate(:status, account: account, uri: uri) + duplicate = Fabricate.build(:status, account: account, uri: uri) + duplicate.save(validate: false) + Fabricate(:status_pin, account: account, status: duplicate) + Fabricate(:status, in_reply_to_id: duplicate.id) + Fabricate(:status, reblog_of_id: duplicate.id) + end + end + + context 'with duplicate tags' do + before do + prepare_duplicate_data + end + + let(:name) { 'tagname' } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating tags', + 'Restoring tags indexes', + 'Finished!' + ) + .and change(duplicate_tags, :count).from(2).to(1) + end + + def duplicate_tags + Tag.where(name: name) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :tags, name: 'index_tags_on_name_lower_btree' + Fabricate(:tag, name: name) + Fabricate.build(:tag, name: name).save(validate: false) + end + end + + context 'with duplicate webauthn_credentials' do + before do + prepare_duplicate_data + end + + let(:external_id) { '123_123_123' } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating webauthn_credentials', + 'Restoring webauthn_credentials indexes', + 'Finished!' + ) + .and change(duplicate_webauthn_credentials, :count).from(2).to(1) + end + + def duplicate_webauthn_credentials + WebauthnCredential.where(external_id: external_id) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :webauthn_credentials, :external_id + Fabricate(:webauthn_credential, external_id: external_id) + Fabricate.build(:webauthn_credential, external_id: external_id).save(validate: false) + end + end + + context 'with duplicate webhooks' do + before do + prepare_duplicate_data + end + + let(:url) { 'https://example.host/path' } + + it 'runs the deduplication process' do + expect { subject } + .to output_results( + 'Deduplicating webhooks', + 'Restoring webhooks indexes', + 'Finished!' + ) + .and change(duplicate_webhooks, :count).from(2).to(1) + end + + def duplicate_webhooks + Webhook.where(url: url) + end + + def prepare_duplicate_data + ActiveRecord::Base.connection.remove_index :webhooks, :url + Fabricate(:webhook, url: url) + Fabricate.build(:webhook, url: url).save(validate: false) + end + end + def agree_to_backup_warning allow(cli.shell) .to receive(:yes?)