Multiple PDS searches during import

app/jobs/commit_patient_changesets_job.rb

@@ -20,6 +20,8 @@ def perform(import)
           import_patients_and_parents(changesets, import)
 
           import_school_moves(changesets, import)
+
+          import_pds_search_results(changesets, import)
         end
 
       import.postprocess_rows!
@@ -96,6 +98,29 @@ def import_school_moves(changesets, import)
     )
   end
 
+  def import_pds_search_results(changesets, import)
+    pds_search_records = []
+
+    changesets.each do |changeset|
+      next if changeset.search_results.blank?
+
+      patient = changeset.patient
+      next unless patient.persisted?
+
+      changeset.search_results.each do |result|
+        pds_search_records << PDSSearchResult.new(
+          patient_id: patient.id,
+          step: PDSSearchResult.steps[result["step"]],
+          result: PDSSearchResult.results[result["result"]],
+          nhs_number: result["nhs_number"],
+          import:
+        )
+      end
+    end
+
+    PDSSearchResult.import(pds_search_records, on_duplicate_key_ignore: true)
+  end
+
   def link_records_to_import(import_source, record_class, records)
     source_type = import_source.class.name
     record_type = record_class.name

app/jobs/patient_nhs_number_lookup_job.rb

@@ -26,9 +26,22 @@ def perform(patient)
        )
       PatientMerger.call(to_keep: existing_patient, to_destroy: patient)
       existing_patient.update_from_pds!(pds_patient)
+
+      PDSSearchResult.create!(
+        patient_id: existing_patient.id,
+        step: :no_fuzzy_with_history_daily,
+        result: :one_match,
+        nhs_number: pds_patient.nhs_number
+      )
     else
       patient.nhs_number = pds_patient.nhs_number
       patient.update_from_pds!(pds_patient)
+      PDSSearchResult.create!(
+        patient_id: patient.id,
+        step: :no_fuzzy_with_history_daily,
+        result: :one_match,
+        nhs_number: pds_patient.nhs_number
+      )
     end
   end
 end

app/jobs/process_patient_changesets_job.rb

@@ -5,15 +5,154 @@ class ProcessPatientChangesetsJob < ApplicationJob
 
   queue_as :imports
 
-  def perform(patient_changeset)
-    attrs = patient_changeset.child_attributes
+  def perform(patient_changeset, step_name = nil)
+    step_name ||= first_step_name
+    step = steps[step_name]
 
-    pds_patient = search_for_patient(attrs)
-    if pds_patient.present?
-      attrs["nhs_number"] = pds_patient.nhs_number
-      patient_changeset.pds_nhs_number = pds_patient.nhs_number
+    result, pds_patient =
+      search_for_patient(patient_changeset.child_attributes, step_name)
+    patient_changeset.search_results << {
+      step: step_name,
+      result: result,
+      nhs_number: pds_patient&.nhs_number
+    }
+
+    if multiple_nhs_numbers_found?(patient_changeset)
+      finish_processing(patient_changeset)
+    end
+
+    next_step = step[result]
+
+    if next_step == :give_up
+      finish_processing(patient_changeset)
+    elsif next_step == :save_nhs_number_if_unique
+      if nhs_number_is_unique_across_searches?(patient_changeset)
+        unique_nhs_number = get_unique_nhs_number(patient_changeset)
+        if unique_nhs_number
+          patient_changeset.child_attributes["nhs_number"] = unique_nhs_number
+          patient_changeset.pds_nhs_number = unique_nhs_number
+        end
+      end
+      finish_processing(patient_changeset)
+    elsif next_step.in?(steps.keys)
+      raise "Recursive step detected: #{next_step}" if next_step == step_name
+      enqueue_next_search(patient_changeset, next_step)
+    elsif result == :no_postcode
+      finish_processing(patient_changeset)
+    else
+      raise "Unknown step: #{next_step}"
+    end
+  end
+
+  private
+
+  def unique_nhs_numbers(patient_changeset)
+    patient_changeset.search_results.pluck(:nhs_number).compact.uniq
+  end
+
+  def get_unique_nhs_number(patient_changeset)
+    numbers = unique_nhs_numbers(patient_changeset)
+    numbers.count == 1 ? numbers.first : nil
+  end
+
+  def nhs_number_is_unique_across_searches?(patient_changeset)
+    unique_nhs_numbers(patient_changeset).count == 1
+  end
+
+  def multiple_nhs_numbers_found?(patient_changeset)
+    unique_nhs_numbers(patient_changeset).count > 1
+  end
+
+  def first_step_name
+    :no_fuzzy_with_history
+  end
+
+  def steps
+    {
+      no_fuzzy_with_history: {
+        no_matches: :no_fuzzy_with_wildcard_postcode,
+        one_match: :save_nhs_number_if_unique,
+        too_many_matches: :no_fuzzy_without_history
+      },
+      no_fuzzy_without_history: {
+        no_matches: :fuzzy_without_history,
+        one_match: :save_nhs_number_if_unique,
+        too_many_matches: :give_up,
+        format_query: ->(query) { query.merge(history: false) }
+      },
+      no_fuzzy_with_wildcard_postcode: {
+        no_matches: :no_fuzzy_with_wildcard_given_name,
+        one_match: :no_fuzzy_with_wildcard_given_name,
+        too_many_matches: :no_fuzzy_with_wildcard_given_name,
+        format_query: ->(query) { query[:address_postcode][2..] = "*" }
+      },
+      no_fuzzy_with_wildcard_given_name: {
+        no_matches: :no_fuzzy_with_wildcard_family_name,
+        one_match: :no_fuzzy_with_wildcard_family_name,
+        too_many_matches: :no_fuzzy_with_wildcard_family_name,
+        format_query: ->(query) { query[:given_name][3..] = "*" }
+      },
+      no_fuzzy_with_wildcard_family_name: {
+        no_matches: :fuzzy_without_history,
+        one_match: :fuzzy_without_history,
+        too_many_matches: :fuzzy_without_history,
+        format_query: ->(query) { query[:family_name][3..] = "*" }
+      },
+      fuzzy_without_history: {
+        no_matches: :fuzzy_with_history,
+        one_match: :save_nhs_number_if_unique,
+        too_many_matches: :save_nhs_number_if_unique,
+        format_query: ->(query) do
+          query[:fuzzy] = true
+          query[:history] = false
+        end
+      },
+      fuzzy_with_history: {
+        no_matches: :give_up,
+        one_match: :save_nhs_number_if_unique,
+        too_many_matches: :save_nhs_number_if_unique,
+        format_query: ->(query) do
+          query[:fuzzy] = true
+          query[:history] = true
+        end
+      }
+    }
+  end
+
+  def search_for_patient(attrs, step_name)
+    return :no_postcode, nil if attrs["address_postcode"].blank?
+    query = {
+      family_name: attrs["family_name"].dup,
+      given_name: attrs["given_name"].dup,
+      date_of_birth: attrs["date_of_birth"].dup,
+      address_postcode: attrs["address_postcode"].dup,
+      history: true,
+      fuzzy: false
+    }
+    if steps[step_name][:format_query].respond_to?(:call)
+      result = steps[step_name][:format_query].call(query)
+      query = result if result.is_a?(Hash)
+    end
+
+    patient = PDS::Patient.search(**query)
+    return :no_matches, nil if patient.nil?
+
+    [:one_match, patient]
+  rescue NHS::PDS::PatientNotFound
+    [:no_matches, nil]
+  rescue NHS::PDS::TooManyMatches
+    [:too_many_matches, nil]
+  end
+
+  def enqueue_next_search(patient_changeset, next_step)
+    if patient_changeset.import.slow?
+      ProcessPatientChangesetsJob.perform_later(patient_changeset, next_step)
+    else
+      ProcessPatientChangesetsJob.perform_now(patient_changeset, next_step)
     end
+  end
 
+  def finish_processing(patient_changeset)
     patient_changeset.processed!
     patient_changeset.save!
 
@@ -26,18 +165,4 @@ def perform(patient_changeset)
       end
     end
   end
-
-  private
-
-  def search_for_patient(attrs)
-    return nil if attrs["address_postcode"].blank?
-    PDS::Patient.search(
-      family_name: attrs["family_name"],
-      given_name: attrs["given_name"],
-      date_of_birth: attrs["date_of_birth"],
-      address_postcode: attrs["address_postcode"]
-    )
-  rescue NHS::PDS::PatientNotFound, NHS::PDS::TooManyMatches
-    nil
-  end
 end

app/models/class_import.rb

@@ -46,6 +46,7 @@ class ClassImport < PatientImport
            class_name: "PatientChangeset",
            as: :import,
            dependent: :destroy
+  has_many :pds_search_results
 
   def postprocess_rows!
     # Remove patients already in the sessions but not in the class list.

app/models/class_imports_patient.rb

@@ -4,8 +4,8 @@
 #
 # Table name: class_imports_patients
 #
-#  class_import_id       :bigint           not null
-#  patient_id            :bigint           not null
+#  class_import_id :bigint           not null
+#  patient_id      :bigint           not null
 #
 # Indexes
 #

app/models/cohort_import.rb

@@ -40,6 +40,7 @@ class CohortImport < PatientImport
            class_name: "PatientChangeset",
            as: :import,
            dependent: :destroy
+  has_many :pds_search_results
 
   def postprocess_rows!
     PatientsAgedOutOfSchoolJob.perform_later

app/models/cohort_imports_patient.rb

@@ -4,8 +4,8 @@
 #
 # Table name: cohort_imports_patients
 #
-#  cohort_import_id      :bigint           not null
-#  patient_id            :bigint           not null
+#  cohort_import_id :bigint           not null
+#  patient_id       :bigint           not null
 #
 # Indexes
 #

app/models/patient.rb

@@ -69,6 +69,7 @@ class Patient < ApplicationRecord
   has_many :notify_log_entries
   has_many :parent_relationships, -> { order(:created_at) }
   has_many :patient_sessions
+  has_many :pds_search_results
   has_many :school_move_log_entries
   has_many :school_moves
   has_many :session_notifications

app/models/patient_changeset.rb

@@ -39,10 +39,8 @@ class PatientChangeset < ApplicationRecord
               },
               parent_2: {
               },
-              pds: {
-              }
+              search_results: []
             }
-
   belongs_to :import, polymorphic: true
   belongs_to :patient, optional: true
   belongs_to :school, class_name: "Location", optional: true
@@ -70,7 +68,8 @@ def self.from_import_row(row:, import:, row_number:)
         #       Maybe one day.
         school_move_source: row.school_move_source,
         parent_1: row.parent_1_import_attributes,
-        parent_2: row.parent_2_import_attributes
+        parent_2: row.parent_2_import_attributes,
+        search_results: []
       }
     )
   end
@@ -83,6 +82,8 @@ def parent_1_attributes = pending_changes["parent_1"]
 
   def parent_2_attributes = pending_changes["parent_2"]
 
+  def search_results = pending_changes["search_results"]
+
   def academic_year = pending_changes["academic_year"]
 
   def home_educated = pending_changes["home_educated"]

app/models/pds/patient.rb

@@ -16,13 +16,21 @@ def find(nhs_number)
       from_pds_fhir_response(response.body)
     end
 
-    def search(family_name:, given_name:, date_of_birth:, address_postcode:)
+    def search(
+      family_name:,
+      given_name:,
+      date_of_birth:,
+      address_postcode:,
+      history: true,
+      fuzzy: false
+    )
       query = {
         "family" => family_name,
         "given" => given_name,
         "birthdate" => "eq#{date_of_birth}",
         "address-postalcode" => address_postcode,
-        "_history" => true # look up previous names and addresses,
+        "_history" => history, # look up previous names and addresses,
+        "_fuzzy-match" => fuzzy
       }.compact_blank
 
       results = NHS::PDS.search_patients(query).body

app/models/pds_search_result.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: pds_search_results
+#
+#  id          :bigint           not null, primary key
+#  import_type :string
+#  nhs_number  :string
+#  result      :integer          not null
+#  step        :integer          not null
+#  created_at  :datetime         not null
+#  updated_at  :datetime         not null
+#  import_id   :bigint
+#  patient_id  :bigint           not null
+#
+# Indexes
+#
+#  index_pds_search_results_on_import               (import_type,import_id)
+#  index_pds_search_results_on_patient_id           (patient_id)
+#  index_pds_search_results_on_patient_import_step  (patient_id,import_type,import_id,step) UNIQUE
+#
+# Foreign Keys
+#
+#  fk_rails_...  (patient_id => patients.id)
+#
+class PDSSearchResult < ApplicationRecord
+  belongs_to :patient
+  belongs_to :import, polymorphic: true, optional: true
+
+  enum :step,
+       {
+         no_fuzzy_with_history: 0,
+         no_fuzzy_with_history_daily: 1,
+         no_fuzzy_without_history: 2,
+         no_fuzzy_with_wildcard_postcode: 3,
+         no_fuzzy_with_wildcard_given_name: 4,
+         no_fuzzy_with_wildcard_family_name: 5,
+         fuzzy_without_history: 6,
+         fuzzy_with_history: 7
+       },
+       validate: true
+
+  enum :result,
+       { no_matches: 0, one_match: 1, too_many_matches: 2 },
+       validate: true
+end