Remove finder tracking from specialist-finder index

The new index should only track specialist documents, as the finder itself is not searchable within the finder and is only needed on govuk wide search

lib/govuk_index/presenters/common_fields_presenter.rb

@@ -1,6 +1,7 @@
 module GovukIndex
   class CommonFieldsPresenter
     CUSTOM_FORMAT_MAP = {
+      "esi_fund" => "european_structural_investment_fund",
       "external_content" => "recommended-link",
       "service_manual_homepage" => "service_manual_guide",
       "service_manual_service_standard" => "service_manual_guide",

lib/specialist_finder_index/publishing_event_processor.rb

@@ -4,7 +4,7 @@ def process(messages)
       messages = Array(messages) # treat a single message as an array with one value
 
       Services.statsd_client.increment("specialist_finder_index.rabbit-mq-consumed")
-      PublishingEventJob.perform_async(messages.map { |msg| [msg.delivery_info[:routing_key], msg.payload] })
+      SpecialistFinderIndex::PublishingEventJob.perform_async(messages.map { |msg| [msg.delivery_info[:routing_key], msg.payload] })
       messages.each(&:ack)
     end
   end

lib/tasks/message_queue.rake

@@ -9,9 +9,8 @@ namespace :message_queue do
     exch = Bunny::Exchange.new(channel, :topic, "published_documents")
     channel.queue("search_api_to_be_indexed").bind(exch, routing_key: "*.links")
     channel.queue("search_api_bulk_reindex").bind(exch, routing_key: "*.bulk.reindex")
-    channel.queue("search_api_govuk_index").bind(exch, routing_key: "*.*")
     channel.queue("search_api_specialist_finder_index_documents").bind(exch, routing_key: "specialist_document.*")
-    channel.queue("search_api_specialist_finder_index_finders").bind(exch, routing_key: "finder.*")
+    channel.queue("search_api_govuk_index").bind(exch, routing_key: "*.*")
   end
 
   desc "Index documents that are published to the publishing-api"
@@ -36,10 +35,6 @@ namespace :message_queue do
       queue_name: "search_api_specialist_finder_index_documents",
       processor: SpecialistFinderIndex::PublishingEventProcessor.new,
     ).run
-    GovukMessageQueueConsumer::Consumer.new(
-      queue_name: "search_api_specialist_finder_index_finders",
-      processor: SpecialistFinderIndex::PublishingEventProcessor.new,
-    ).run
   end
 
   desc "Gets data from RabbitMQ and insert into govuk index (bulk reindex queue)"

spec/integration/govuk_index/specialist_formats_spec.rb

@@ -0,0 +1,101 @@
+require "spec_helper"
+RSpec.describe "SpecialistFormatsTest" do
+  before do
+    bunny_mock = BunnyMock.new
+    @channel = bunny_mock.start.channel
+
+    consumer = GovukMessageQueueConsumer::Consumer.new(
+      queue_name: "bigwig.test",
+      processor: GovukIndex::PublishingEventProcessor.new,
+      rabbitmq_connection: bunny_mock,
+    )
+
+    @queue = @channel.queue("bigwig.test")
+    consumer.run
+  end
+
+  it "specialist publisher finders are correctly indexed" do
+    random_example = generate_random_example(
+      schema: "finder",
+      payload: { document_type: "finder" },
+    )
+
+    allow(GovukIndex::MigratedFormats).to receive(:indexable_formats).and_return("finder" => :all)
+
+    @queue.publish(random_example.to_json, content_type: "application/json")
+
+    expect_document_is_in_rummager({ "link" => random_example["base_path"] }, index: "govuk_test", type: "edition")
+  end
+
+  it "specialist documents are correctly indexed" do
+    document_types = %w[
+      aaib_report
+      asylum_support_decision
+      business_finance_support_scheme
+      cma_case
+      countryside_stewardship_grant
+      drug_safety_update
+      employment_appeal_tribunal_decision
+      employment_tribunal_decision
+      flood_and_coastal_erosion_risk_management_research_report
+      international_development_fund
+      licence_transaction
+      maib_report
+      medical_safety_alert
+      protected_food_drink_name
+      raib_report
+      research_for_development_output
+      residential_property_tribunal_decision
+      service_standard_report
+      statutory_instrument
+      tax_tribunal_decision
+      utaac_decision
+    ]
+
+    # ideally we would run a test for all document types, but this takes 3 seconds so I have limited
+    # it to a random subset
+    document_types.sample(3).each do |specialist_document_type|
+      random_example = generate_random_example(
+        schema: "specialist_document",
+        payload: { document_type: specialist_document_type },
+      )
+      allow(GovukIndex::MigratedFormats).to receive(:indexable_formats).and_return(specialist_document_type => :all)
+
+      @queue.publish(random_example.to_json, content_type: "application/json")
+
+      expect_document_is_in_rummager({ "link" => random_example["base_path"] }, index: "govuk_test", type: specialist_document_type)
+    end
+  end
+
+  it "esi documents are correctly indexed" do
+    publisher_document_type = "esi_fund"
+    search_document_type = "european_structural_investment_fund"
+
+    random_example = generate_random_example(
+      schema: "specialist_document",
+      payload: { document_type: publisher_document_type },
+    )
+    allow(GovukIndex::MigratedFormats).to receive(:indexable_formats).and_return(search_document_type => :all)
+
+    @queue.publish(random_example.to_json, content_type: "application/json")
+
+    expect_document_is_in_rummager(
+      { "link" => random_example["base_path"], "format" => search_document_type },
+      index: "govuk_test",
+      type: search_document_type,
+    )
+  end
+
+  it "finders email signup are never indexed" do
+    random_example = generate_random_example(
+      schema: "finder_email_signup",
+      payload: { document_type: "finder_email_signup" },
+    )
+
+    @queue.publish(random_example.to_json, content_type: "application/json")
+
+    expect {
+      fetch_document_from_rummager(id: random_example["base_path"], index: "govuk_test")
+    }.to raise_error(Elasticsearch::Transport::Transport::Errors::NotFound)
+  end
+end

spec/integration/specialist_finder_index/specialist_formats_spec.rb → spec/integration/specialist_finder_index/specialist_documents_spec.rb

@@ -1,6 +1,6 @@
 require "spec_helper"
 
-RSpec.describe "SpecialistFormatTest" do
+RSpec.describe "SpecialistDocumentsTest" do
   before do
     bunny_mock = BunnyMock.new
     @channel = bunny_mock.start.channel
@@ -15,17 +15,6 @@
     consumer.run
   end
 
-  it "specialist publisher finders are correctly indexed" do
-    random_example = generate_random_example(
-      schema: "finder",
-      payload: { document_type: "finder" },
-    )
-
-    @queue.publish(random_example.to_json, content_type: "application/json")
-
-    expect_document_is_in_rummager({ "link" => random_example["base_path"] }, index: "specialist-finder_test", type: "finder")
-  end
-
   it "specialist documents are correctly indexed" do
     document_types = %w[
       aaib_report
@@ -49,7 +38,6 @@
       statutory_instrument
       tax_tribunal_decision
       utaac_decision
-      esi_fund
     ]
 
     # ideally we would run a test for all document types, but this takes 3 seconds so I have limited
@@ -65,4 +53,22 @@
       expect_document_is_in_rummager({ "link" => random_example["base_path"] }, index: "specialist-finder_test", type: specialist_document_type)
     end
   end
+
+  it "esi documents are correctly indexed" do
+    publisher_document_type = "esi_fund"
+    search_document_type = "european_structural_investment_fund"
+
+    random_example = generate_random_example(
+      schema: "specialist_document",
+      payload: { document_type: publisher_document_type },
+    )
+
+    @queue.publish(random_example.to_json, content_type: "application/json")
+
+    expect_document_is_in_rummager(
+      { "link" => random_example["base_path"], "format" => search_document_type },
+      index: "specialist-finder_test",
+      type: publisher_document_type,
+    )
+  end
 end