Duplicate schools fix

app/models/data_hub/duplicate_schools_process_summary.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module DataHub
+  # Stores what a duplicate schools classification run found. Nothing is
+  # changed by the run, so this is the whole of its output: the counts a merge
+  # policy gets chosen from, and the evidence behind them.
+  #
+  # Kinds and flags are tallies rather than a field per kind, so the list in
+  # DataHub::DuplicateSchools::Kind stays the single source of truth and a fifth
+  # shape needs no migration.
+  class DuplicateSchoolsProcessSummary < ProcessSummary
+    jsonb_accessor :short_summary,
+                   years: [:string, { array: true, default: [] }],
+                   groups_processed: :integer,
+                   surplus_sites: :integer,
+                   surplus_provider_schools: :integer,
+                   kinds: [:jsonb, { array: true, default: [] }],
+                   flags: [:jsonb, { array: true, default: [] }]
+
+    jsonb_accessor :full_summary,
+                   duplicate_groups: [:jsonb, { array: true, default: [] }]
+  end
+end

app/services/data_hub/duplicate_schools/classifier.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module DataHub
+  module DuplicateSchools
+    # Finds the (provider, urn) pairs held by more than one live school site and
+    # returns them as classified groups. Reads only.
+    #
+    # Deliberately excluded from the grouping key: `code`. The query this
+    # replaces filtered `code <> '-'`, which hid the largest shape - a
+    # provider's main site colliding with the same school added again as a
+    # placement school.
+    class Classifier
+      def initialize(years:)
+        @years = Array(years).map(&:to_s)
+      end
+
+      # @return [Array<Group>]
+      def call
+        keys = duplicate_keys
+        return [] if keys.empty?
+
+        sites = sites_for(keys).group_by { |site| [site.provider_id, site.urn] }
+        gias_schools = GiasSchool.where(urn: keys.map(&:last).uniq).index_by(&:urn)
+        provider_schools = provider_schools_for(keys.map(&:first).uniq, gias_schools.values)
+
+        keys.filter_map do |key|
+          group_sites = sites[key]
+          next if group_sites.blank?
+
+          provider = group_sites.first.provider
+          gias_school = gias_schools[key.last]
+
+          Group.new(
+            year: provider.recruitment_cycle.year,
+            provider:,
+            urn: key.last,
+            sites: group_sites.sort_by(&:id),
+            gias_school:,
+            provider_schools: provider_schools[[provider.id, gias_school&.id]].to_a,
+          )
+        end
+      end
+
+    private
+
+      attr_reader :years
+
+      def duplicate_keys
+        Site.kept
+            .school
+            .where.not(urn: [nil, ""])
+            .joins(provider: :recruitment_cycle)
+            .where(provider: { discarded_at: nil }, recruitment_cycle: { year: years })
+            .group(:provider_id, :urn)
+            .having("COUNT(*) > 1")
+            .pluck(:provider_id, :urn)
+      end
+
+      def sites_for(keys)
+        Site.kept
+            .school
+            .where(provider_id: keys.map(&:first).uniq, urn: keys.map(&:last).uniq)
+            .includes(:site_statuses, provider: :recruitment_cycle)
+            .select { |site| keys.include?([site.provider_id, site.urn]) }
+      end
+
+      def provider_schools_for(provider_ids, gias_schools)
+        Provider::School
+          .where(provider_id: provider_ids, gias_school_id: gias_schools.map(&:id))
+          .includes(:course_schools)
+          .group_by { |provider_school| [provider_school.provider_id, provider_school.gias_school_id] }
+      end
+    end
+  end
+end

app/services/data_hub/duplicate_schools/executor.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module DataHub
+  module DuplicateSchools
+    # Classifies the providers holding the same school twice and records what it
+    # found in the process summary table.
+    #
+    #   DataHub::DuplicateSchools::Executor.new(years: %w[2026]).execute
+    #
+    # Nothing about the duplicates is changed - which shape gets merged, and
+    # onto which row, is a decision to take from the recorded evidence. The only
+    # row this writes is its own summary.
+    class Executor
+      # @param years [Array<String>] recruitment cycle years to classify
+      # @param io [IO] where the human-readable report is printed
+      def initialize(years:, io: $stdout)
+        @years = Array(years).map(&:to_s)
+        @io = io
+      end
+
+      # @return [DataHub::DuplicateSchoolsProcessSummary]
+      def execute
+        process_summary = DataHub::DuplicateSchoolsProcessSummary.start!
+
+        groups = Classifier.new(years:).call
+        summary_builder = SummaryBuilder.new(groups:, years:)
+
+        Reporter.new(groups:, years:, io:).call
+
+        process_summary.finish!(
+          short_summary: summary_builder.short_summary,
+          full_summary: summary_builder.full_summary,
+        )
+
+        process_summary
+      rescue StandardError => e
+        process_summary&.fail!(e)
+        raise e
+      end
+
+    private
+
+      attr_reader :years, :io
+    end
+  end
+end

app/services/data_hub/duplicate_schools/group.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module DataHub
+  module DuplicateSchools
+    # One provider's live school sites sharing a urn, with the GIAS record they
+    # all point at and the Provider::School rows they became - the pair the
+    # schools list and the course pickers actually read.
+    Group = Struct.new(:year, :provider, :urn, :sites, :gias_school, :provider_schools, keyword_init: true) do
+      def self.normalise(value)
+        value.to_s.strip.downcase.squeeze(" ")
+      end
+
+      def kind
+        @kind ||= Kind.for(self)
+      end
+
+      def codes
+        sites.map(&:code).uniq
+      end
+
+      def names
+        sites.map { |site| Group.normalise(site.location_name) }.uniq
+      end
+
+      def gias_name
+        Group.normalise(gias_school&.name)
+      end
+
+      def main_site
+        sites.find { |site| site.code == Provider::School::MAIN_SITE_CODE }
+      end
+
+      # What DataHub::Sites::Deduplication::Deduplicator#pick_primary_site would
+      # keep, so the report can say whose row that heuristic would discard.
+      def legacy_primary
+        sites.max_by { |site| [course_ids(site).size, -site.id] }
+      end
+
+      def course_ids(site)
+        site.site_statuses.map(&:course_id).uniq
+      end
+
+      def unique_course_ids(site)
+        course_ids(site) - sites.reject { |other| other == site }.flat_map { |other| course_ids(other) }
+      end
+
+      def sites_with_courses
+        sites.count { |site| course_ids(site).any? }
+      end
+
+      def surplus
+        sites.size - 1
+      end
+
+      def provider_school_surplus
+        [provider_schools.size - 1, 0].max
+      end
+    end
+  end
+end

app/services/data_hub/duplicate_schools/kind.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module DataHub
+  module DuplicateSchools
+    # A shape of duplicate. Subclasses declare how to recognise themselves, what
+    # to say about themselves, and which flags matter to the decision they need.
+    # Splitting a shape in two later is a new subclass, not a new branch.
+    class Kind
+      # Matching order is precedence, declared rather than left to the order
+      # Zeitwerk happens to load the subclasses in: a '-' row paired with an
+      # identical clone is still a main site collision, and DivergentNameTwin is
+      # the terminal fallback.
+      ORDER = %w[MainSiteCollision Clone SplitCodeTwin DivergentNameTwin].freeze
+
+      class << self
+        def for(group)
+          ORDER.lazy.map { |name| const_get(name).new(group) }.find(&:matches?)
+        end
+
+        def matches(&block)
+          define_method(:matches?, &block)
+        end
+
+        def headline(text)
+          define_method(:headline) { text }
+        end
+
+        def action(text)
+          define_method(:suggested_action) { text }
+        end
+
+        def flag(name, &block)
+          declared_flags << name
+          define_method(:"#{name}?", &block)
+        end
+
+        def declared_flags
+          @declared_flags ||= []
+        end
+
+        def flags
+          inherited_flags = superclass.respond_to?(:flags) ? superclass.flags : []
+          inherited_flags + declared_flags
+        end
+      end
+
+      attr_reader :group
+
+      delegate :codes, :names, :sites, :main_site, :gias_school, :gias_name,
+               :legacy_primary, :sites_with_courses, :provider, :urn, to: :group
+
+      def initialize(group)
+        @group = group
+      end
+
+      def label
+        self.class.name.demodulize.underscore
+      end
+
+      def flags
+        self.class.flags.index_with { |name| public_send(:"#{name}?") }
+      end
+
+      def raised_flags
+        flags.select { |_name, raised| raised }.keys
+      end
+
+      flag(:gias_closed) { gias_school.nil? || gias_school.closed? }
+    end
+  end
+end

app/services/data_hub/duplicate_schools/kind/clone.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module DataHub
+  module DuplicateSchools
+    class Kind
+      # The same school added repeatedly under one code. Provider::School's
+      # unique (provider_id, gias_school_id, site_code) index already collapsed
+      # these, so they are legacy litter rather than anything a provider sees.
+      class Clone < Kind
+        matches { codes.one? && names.one? }
+        headline "the same school added repeatedly under one code"
+        action "safe to merge unattended - no user-visible duplicate to remove"
+      end
+    end
+  end
+end

app/services/data_hub/duplicate_schools/kind/divergent_name_twin.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module DataHub
+  module DuplicateSchools
+    class Kind
+      # Same urn, two names a provider wrote. Often deliberate labelling rather
+      # than an accident, so it needs asking rather than merging.
+      class DivergentNameTwin < Kind
+        matches { true }
+        headline "one urn under two provider-written names"
+        action "ask the provider which name they meant before merging"
+
+        # A name that is neither the GIAS name nor the generic main site label
+        # is one the provider typed for themselves, so it carries meaning worth
+        # asking about - "Main site Secondary- one of our partner schools"
+        # counts, a plain "Main Site" does not.
+        flag(:provider_authored_name) do
+          names.any? { |name| name != gias_name && name != Site::MAIN_SITE }
+        end
+        flag(:courses_on_both_sides) { sites_with_courses > 1 }
+      end
+    end
+  end
+end

app/services/data_hub/duplicate_schools/kind/main_site_collision.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module DataHub
+  module DuplicateSchools
+    class Kind
+      # The provider's main site and the same school added again as a placement
+      # school - the shape the original query's `code <> '-'` filter hid. Which
+      # row should survive is a policy decision, so nothing here assumes one.
+      class MainSiteCollision < Kind
+        matches { codes.include?(Provider::School::MAIN_SITE_CODE) }
+        headline "the provider's main site and the same school added again as a placement school"
+        action "decide the policy: keep '-' and move courses onto it, or keep the named row"
+
+        flag(:main_site_at_risk) { legacy_primary.code != Provider::School::MAIN_SITE_CODE }
+        flag(:courses_on_both_sides) { sites_with_courses > 1 }
+      end
+    end
+  end
+end

app/services/data_hub/duplicate_schools/kind/split_code_twin.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module DataHub
+  module DuplicateSchools
+    class Kind
+      # One school under two codes: two Provider::School rows, so the provider
+      # sees the same name twice with its courses split between them.
+      class SplitCodeTwin < Kind
+        matches { names.one? }
+        headline "one school held under two codes - listed twice, courses split"
+        action "merge onto the code holding more courses, moving the rest across"
+
+        flag(:courses_on_both_sides) { sites_with_courses > 1 }
+      end
+    end
+  end
+end