From 41a059393e88762451a884ec809d168011ee8f5e Mon Sep 17 00:00:00 2001 From: Minno Dang Date: Wed, 30 Oct 2024 17:32:20 +0000 Subject: [PATCH] Remove finder tracking from specialist-finder index The new index should only track specialist documents, as the finder itself is not searchable within the finder and is only needed on govuk wide search --- .../presenters/common_fields_presenter.rb | 1 + .../publishing_event_processor.rb | 2 +- lib/tasks/message_queue.rake | 7 +- .../govuk_index/specialist_formats_spec.rb | 101 ++++++++++++++++++ ...s_spec.rb => specialist_documents_spec.rb} | 32 +++--- .../specialist_formats_spec.rb | 0 6 files changed, 123 insertions(+), 20 deletions(-) create mode 100644 spec/integration/govuk_index/specialist_formats_spec.rb rename spec/integration/specialist_finder_index/{specialist_formats_spec.rb => specialist_documents_spec.rb} (78%) rename spec/unit/specialist_finder_index/{ => presenter}/specialist_formats_spec.rb (100%) diff --git a/lib/govuk_index/presenters/common_fields_presenter.rb b/lib/govuk_index/presenters/common_fields_presenter.rb index ede465646..f532ab6e0 100644 --- a/lib/govuk_index/presenters/common_fields_presenter.rb +++ b/lib/govuk_index/presenters/common_fields_presenter.rb @@ -1,6 +1,7 @@ module GovukIndex class CommonFieldsPresenter CUSTOM_FORMAT_MAP = { + "esi_fund" => "european_structural_investment_fund", "external_content" => "recommended-link", "service_manual_homepage" => "service_manual_guide", "service_manual_service_standard" => "service_manual_guide", diff --git a/lib/specialist_finder_index/publishing_event_processor.rb b/lib/specialist_finder_index/publishing_event_processor.rb index 6e21c26d7..158200431 100644 --- a/lib/specialist_finder_index/publishing_event_processor.rb +++ b/lib/specialist_finder_index/publishing_event_processor.rb @@ -4,7 +4,7 @@ def process(messages) messages = Array(messages) # treat a single message as an array with one value Services.statsd_client.increment("specialist_finder_index.rabbit-mq-consumed") - PublishingEventJob.perform_async(messages.map { |msg| [msg.delivery_info[:routing_key], msg.payload] }) + SpecialistFinderIndex::PublishingEventJob.perform_async(messages.map { |msg| [msg.delivery_info[:routing_key], msg.payload] }) messages.each(&:ack) end end diff --git a/lib/tasks/message_queue.rake b/lib/tasks/message_queue.rake index 0743438c0..06a935b4d 100644 --- a/lib/tasks/message_queue.rake +++ b/lib/tasks/message_queue.rake @@ -9,9 +9,8 @@ namespace :message_queue do exch = Bunny::Exchange.new(channel, :topic, "published_documents") channel.queue("search_api_to_be_indexed").bind(exch, routing_key: "*.links") channel.queue("search_api_bulk_reindex").bind(exch, routing_key: "*.bulk.reindex") - channel.queue("search_api_govuk_index").bind(exch, routing_key: "*.*") channel.queue("search_api_specialist_finder_index_documents").bind(exch, routing_key: "specialist_document.*") - channel.queue("search_api_specialist_finder_index_finders").bind(exch, routing_key: "finder.*") + channel.queue("search_api_govuk_index").bind(exch, routing_key: "*.*") end desc "Index documents that are published to the publishing-api" @@ -36,10 +35,6 @@ namespace :message_queue do queue_name: "search_api_specialist_finder_index_documents", processor: SpecialistFinderIndex::PublishingEventProcessor.new, ).run - GovukMessageQueueConsumer::Consumer.new( - queue_name: "search_api_specialist_finder_index_finders", - processor: SpecialistFinderIndex::PublishingEventProcessor.new, - ).run end desc "Gets data from RabbitMQ and insert into govuk index (bulk reindex queue)" diff --git a/spec/integration/govuk_index/specialist_formats_spec.rb b/spec/integration/govuk_index/specialist_formats_spec.rb new file mode 100644 index 000000000..d648a1f1a --- /dev/null +++ b/spec/integration/govuk_index/specialist_formats_spec.rb @@ -0,0 +1,101 @@ +require "spec_helper" +RSpec.describe "SpecialistFormatsTest" do + before do + bunny_mock = BunnyMock.new + @channel = bunny_mock.start.channel + + consumer = GovukMessageQueueConsumer::Consumer.new( + queue_name: "bigwig.test", + processor: GovukIndex::PublishingEventProcessor.new, + rabbitmq_connection: bunny_mock, + ) + + @queue = @channel.queue("bigwig.test") + consumer.run + end + + it "specialist publisher finders are correctly indexed" do + random_example = generate_random_example( + schema: "finder", + payload: { document_type: "finder" }, + ) + + allow(GovukIndex::MigratedFormats).to receive(:indexable_formats).and_return("finder" => :all) + + @queue.publish(random_example.to_json, content_type: "application/json") + + expect_document_is_in_rummager({ "link" => random_example["base_path"] }, index: "govuk_test", type: "edition") + end + + it "specialist documents are correctly indexed" do + document_types = %w[ + aaib_report + asylum_support_decision + business_finance_support_scheme + cma_case + countryside_stewardship_grant + drug_safety_update + employment_appeal_tribunal_decision + employment_tribunal_decision + flood_and_coastal_erosion_risk_management_research_report + international_development_fund + licence_transaction + maib_report + medical_safety_alert + protected_food_drink_name + raib_report + research_for_development_output + residential_property_tribunal_decision + service_standard_report + statutory_instrument + tax_tribunal_decision + utaac_decision + ] + + # ideally we would run a test for all document types, but this takes 3 seconds so I have limited + # it to a random subset + document_types.sample(3).each do |specialist_document_type| + random_example = generate_random_example( + schema: "specialist_document", + payload: { document_type: specialist_document_type }, + ) + allow(GovukIndex::MigratedFormats).to receive(:indexable_formats).and_return(specialist_document_type => :all) + + @queue.publish(random_example.to_json, content_type: "application/json") + + expect_document_is_in_rummager({ "link" => random_example["base_path"] }, index: "govuk_test", type: specialist_document_type) + end + end + + it "esi documents are correctly indexed" do + publisher_document_type = "esi_fund" + search_document_type = "european_structural_investment_fund" + + random_example = generate_random_example( + schema: "specialist_document", + payload: { document_type: publisher_document_type }, + ) + allow(GovukIndex::MigratedFormats).to receive(:indexable_formats).and_return(search_document_type => :all) + + @queue.publish(random_example.to_json, content_type: "application/json") + + expect_document_is_in_rummager( + { "link" => random_example["base_path"], "format" => search_document_type }, + index: "govuk_test", + type: search_document_type, + ) + end + + it "finders email signup are never indexed" do + random_example = generate_random_example( + schema: "finder_email_signup", + payload: { document_type: "finder_email_signup" }, + ) + + @queue.publish(random_example.to_json, content_type: "application/json") + + expect { + fetch_document_from_rummager(id: random_example["base_path"], index: "govuk_test") + }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound) + end +end diff --git a/spec/integration/specialist_finder_index/specialist_formats_spec.rb b/spec/integration/specialist_finder_index/specialist_documents_spec.rb similarity index 78% rename from spec/integration/specialist_finder_index/specialist_formats_spec.rb rename to spec/integration/specialist_finder_index/specialist_documents_spec.rb index 21939c780..7ec78419c 100644 --- a/spec/integration/specialist_finder_index/specialist_formats_spec.rb +++ b/spec/integration/specialist_finder_index/specialist_documents_spec.rb @@ -1,6 +1,6 @@ require "spec_helper" -RSpec.describe "SpecialistFormatTest" do +RSpec.describe "SpecialistDocumentsTest" do before do bunny_mock = BunnyMock.new @channel = bunny_mock.start.channel @@ -15,17 +15,6 @@ consumer.run end - it "specialist publisher finders are correctly indexed" do - random_example = generate_random_example( - schema: "finder", - payload: { document_type: "finder" }, - ) - - @queue.publish(random_example.to_json, content_type: "application/json") - - expect_document_is_in_rummager({ "link" => random_example["base_path"] }, index: "specialist-finder_test", type: "finder") - end - it "specialist documents are correctly indexed" do document_types = %w[ aaib_report @@ -49,7 +38,6 @@ statutory_instrument tax_tribunal_decision utaac_decision - esi_fund ] # ideally we would run a test for all document types, but this takes 3 seconds so I have limited @@ -65,4 +53,22 @@ expect_document_is_in_rummager({ "link" => random_example["base_path"] }, index: "specialist-finder_test", type: specialist_document_type) end end + + it "esi documents are correctly indexed" do + publisher_document_type = "esi_fund" + search_document_type = "european_structural_investment_fund" + + random_example = generate_random_example( + schema: "specialist_document", + payload: { document_type: publisher_document_type }, + ) + + @queue.publish(random_example.to_json, content_type: "application/json") + + expect_document_is_in_rummager( + { "link" => random_example["base_path"], "format" => search_document_type }, + index: "specialist-finder_test", + type: publisher_document_type, + ) + end end diff --git a/spec/unit/specialist_finder_index/specialist_formats_spec.rb b/spec/unit/specialist_finder_index/presenter/specialist_formats_spec.rb similarity index 100% rename from spec/unit/specialist_finder_index/specialist_formats_spec.rb rename to spec/unit/specialist_finder_index/presenter/specialist_formats_spec.rb