From ac0eb0533eb97686b2dca6f31d0b87d8c6fd5c31 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 21 Aug 2023 16:50:22 +0200 Subject: [PATCH] Add Elasticsearch cluster health check and indexes mismatch check to dashboard (#26448) --- app/chewy/instances_index.rb | 2 +- .../admin/system_check/elasticsearch_check.rb | 61 ++++++++++++++++++- config/locales/en.yml | 14 +++++ .../system_check/elasticsearch_check_spec.rb | 35 ++++++++++- 4 files changed, 105 insertions(+), 7 deletions(-) diff --git a/app/chewy/instances_index.rb b/app/chewy/instances_index.rb index 0d58167dc..8f10d13b6 100644 --- a/app/chewy/instances_index.rb +++ b/app/chewy/instances_index.rb @@ -6,7 +6,7 @@ class InstancesIndex < Chewy::Index index_scope ::Instance.searchable root date_detection: false do - field :domain, type: 'text', index_prefixes: { min_chars: 1 } + field :domain, type: 'text', index_prefixes: { min_chars: 1, max_chars: 5 } field :accounts_count, type: 'long' end end diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb index 0b55be350..a6f1f164a 100644 --- a/app/lib/admin/system_check/elasticsearch_check.rb +++ b/app/lib/admin/system_check/elasticsearch_check.rb @@ -1,6 +1,13 @@ # frozen_string_literal: true class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck + INDEXES = [ + InstancesIndex, + AccountsIndex, + TagsIndex, + StatusesIndex, + ].freeze + def skip? !current_user.can?(:view_devops) end @@ -8,11 +15,15 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck def pass? return true unless Chewy.enabled? - running_version.present? && compatible_version? + running_version.present? && compatible_version? && cluster_health['status'] == 'green' && indexes_match? && preset_matches? + rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error + false end def message - if running_version.present? + if running_version.blank? + Admin::SystemCheck::Message.new(:elasticsearch_running_check) + elsif !compatible_version? Admin::SystemCheck::Message.new( :elasticsearch_version_check, I18n.t( @@ -21,13 +32,32 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck required_version: required_version ) ) + elsif !indexes_match? + Admin::SystemCheck::Message.new( + :elasticsearch_index_mismatch, + mismatched_indexes.join(' ') + ) + elsif cluster_health['status'] == 'red' + Admin::SystemCheck::Message.new(:elasticsearch_health_red) + elsif cluster_health['number_of_nodes'] < 2 && es_preset != 'single_node_cluster' + Admin::SystemCheck::Message.new(:elasticsearch_preset_single_node, nil, 'https://docs.joinmastodon.org/admin/optional/elasticsearch/#scaling') + elsif Chewy.client.indices.get_settings['chewy_specifications'].dig('settings', 'index', 'number_of_replicas')&.to_i&.positive? && es_preset == 'single_node_cluster' + Admin::SystemCheck::Message.new(:elasticsearch_reset_chewy) + elsif cluster_health['status'] == 'yellow' + Admin::SystemCheck::Message.new(:elasticsearch_health_yellow) else - Admin::SystemCheck::Message.new(:elasticsearch_running_check) + Admin::SystemCheck::Message.new(:elasticsearch_preset, nil, 'https://docs.joinmastodon.org/admin/optional/elasticsearch/#scaling') end + rescue Faraday::ConnectionFailed, Elasticsearch::Transport::Transport::Error + Admin::SystemCheck::Message.new(:elasticsearch_running_check) end private + def cluster_health + @cluster_health ||= Chewy.client.cluster.health + end + def running_version @running_version ||= begin Chewy.client.info['version']['number'] @@ -49,5 +79,30 @@ class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck Gem::Version.new(running_version) >= Gem::Version.new(required_version) || Gem::Version.new(compatible_wire_version) >= Gem::Version.new(required_version) + rescue ArgumentError + false + end + + def mismatched_indexes + @mismatched_indexes ||= INDEXES.filter_map do |klass| + klass.index_name if Chewy.client.indices.get_mapping[klass.index_name]&.deep_symbolize_keys != klass.mappings_hash + end + end + + def indexes_match? + mismatched_indexes.empty? + end + + def es_preset + ENV.fetch('ES_PRESET', 'single_node_cluster') + end + + def preset_matches? + case es_preset + when 'single_node_cluster' + cluster_health['number_of_nodes'] == 1 + else + cluster_health['number_of_nodes'] > 1 + end end end diff --git a/config/locales/en.yml b/config/locales/en.yml index 4a052e000..389b7aa66 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -814,6 +814,20 @@ en: system_checks: database_schema_check: message_html: There are pending database migrations. Please run them to ensure the application behaves as expected + elasticsearch_health_red: + message_html: Elasticsearch cluster is unhealthy (red status), search features are unavailable + elasticsearch_health_yellow: + message_html: Elasticsearch cluster is unhealthy (yellow status), you may want to investigate the reason + elasticsearch_index_mismatch: + message_html: Elasticsearch index mappings are outdated. Please run tootctl search deploy --only=%{value} + elasticsearch_preset: + action: See documentation + message_html: Your Elasticsearch cluster has more than one node, but Mastodon is not configured to use them. + elasticsearch_preset_single_node: + action: See documentation + message_html: Your Elasticsearch cluster has only one node, ES_PRESET should be set to single_node_cluster. + elasticsearch_reset_chewy: + message_html: Your Elasticsearch system index is outdated due to a setting change. Please run tootctl search deploy --reset-chewy to update it. elasticsearch_running_check: message_html: Could not connect to Elasticsearch. Please check that it is running, or disable full-text search elasticsearch_version_check: diff --git a/spec/lib/admin/system_check/elasticsearch_check_spec.rb b/spec/lib/admin/system_check/elasticsearch_check_spec.rb index 498215926..bf518b56e 100644 --- a/spec/lib/admin/system_check/elasticsearch_check_spec.rb +++ b/spec/lib/admin/system_check/elasticsearch_check_spec.rb @@ -11,7 +11,25 @@ describe Admin::SystemCheck::ElasticsearchCheck do describe 'pass?' do context 'when chewy is enabled' do - before { allow(Chewy).to receive(:enabled?).and_return(true) } + before do + allow(Chewy).to receive(:enabled?).and_return(true) + allow(Chewy.client.cluster).to receive(:health).and_return({ 'status' => 'green', 'number_of_nodes' => 1 }) + allow(Chewy.client.indices).to receive(:get_mapping).and_return({ + AccountsIndex.index_name => AccountsIndex.mappings_hash.deep_stringify_keys, + StatusesIndex.index_name => StatusesIndex.mappings_hash.deep_stringify_keys, + InstancesIndex.index_name => InstancesIndex.mappings_hash.deep_stringify_keys, + TagsIndex.index_name => TagsIndex.mappings_hash.deep_stringify_keys, + }) + allow(Chewy.client.indices).to receive(:get_settings).and_return({ + 'chewy_specifications' => { + 'settings' => { + 'index' => { + 'number_of_replicas' => 0, + }, + }, + }, + }) + end context 'when running version is present and high enough' do before do @@ -67,8 +85,19 @@ describe Admin::SystemCheck::ElasticsearchCheck do end describe 'message' do + before do + allow(Chewy).to receive(:enabled?).and_return(true) + allow(Chewy.client.cluster).to receive(:health).and_return({ 'status' => 'green', 'number_of_nodes' => 1 }) + allow(Chewy.client.indices).to receive(:get_mapping).and_return({ + AccountsIndex.index_name => AccountsIndex.mappings_hash.deep_stringify_keys, + StatusesIndex.index_name => StatusesIndex.mappings_hash.deep_stringify_keys, + InstancesIndex.index_name => InstancesIndex.mappings_hash.deep_stringify_keys, + TagsIndex.index_name => TagsIndex.mappings_hash.deep_stringify_keys, + }) + end + context 'when running version is present' do - before { allow(Chewy.client).to receive(:info).and_return({ 'version' => { 'number' => '999.99.9' } }) } + before { allow(Chewy.client).to receive(:info).and_return({ 'version' => { 'number' => '1.2.3' } }) } it 'sends class name symbol to message instance' do allow(Admin::SystemCheck::Message).to receive(:new) @@ -77,7 +106,7 @@ describe Admin::SystemCheck::ElasticsearchCheck do check.message expect(Admin::SystemCheck::Message).to have_received(:new) - .with(:elasticsearch_version_check, 'Elasticsearch 999.99.9 is running while 7.x is required') + .with(:elasticsearch_version_check, 'Elasticsearch 1.2.3 is running while 7.x is required') end end