Merge pull request #23399 from jrafanie/add-purge-orphan-tool

Add purge orphan tool

Author: Fryguy

app/models/mixins/purging_mixin.rb

@@ -116,10 +116,10 @@ def purge_by_scope(older_than = nil, window = nil, &block)
       total
     end
 
-    def purge_by_orphaned(fk_name, window = purge_window_size)
+    def purge_by_orphaned(fk_name, window = purge_window_size, purge_mode = :purge)
       _log.info("Purging orphans in #{table_name}...")
-      total = purge_orphans(fk_name, window)
-      _log.info("Purging orphans in #{table_name}...Complete - Deleted #{total} records")
+      total = purge_orphans(fk_name, window, purge_mode)
+      _log.info("Purging orphans in #{table_name}...Complete - #{purge_mode.to_sym == :count ? 'Would delete' : 'Deleted'} #{total} records")
       total
     end
 
@@ -133,23 +133,38 @@ def purge_by_date_and_orphaned(older_than, fk_name, window = purge_window_size)
 
     private
 
-    def purge_orphans(fk_name, window)
-      # For now, this only supports polymorphic references
-      # We don't currently have a situation where a table with a regular reference needs to be purged
+    def purge_orphans(fk_name, window, purge_mode = :purge)
+      reflection = reflect_on_association(fk_name)
+      scopes = reflection.polymorphic? ? polymorphic_orphan_scopes(fk_name) : non_polymorphic_orphan_scopes(fk_name, reflection.klass)
+
+      if purge_mode.to_sym == :count
+        scopes.sum(&:count)
+      else
+        scopes.sum { |s| purge_in_batches(s, window) }
+      end
+    end
+
+    def polymorphic_orphan_scopes(fk_name)
       polymorphic_type_column = "#{fk_name}_type"
       polymorphic_id_column   = connection.quote_column_name("#{fk_name}_id")
-      total = 0
 
-      polymorphic_classes(polymorphic_type_column).each do |klass|
+      polymorphic_classes(polymorphic_type_column).collect do |klass|
         resource_table = connection.quote_table_name(klass.table_name)
         q_table_name = connection.quote_table_name(table_name)
 
-        scope = joins("LEFT OUTER JOIN #{resource_table} ON #{q_table_name}.#{polymorphic_id_column} = #{resource_table}.id")
-                .where(resource_table => {:id => nil})
-                .where("#{q_table_name}.#{connection.quote_column_name(polymorphic_type_column)} = #{connection.quote(klass.name)}")
-        total += purge_in_batches(scope, window)
+        joins("LEFT OUTER JOIN #{resource_table} ON #{q_table_name}.#{polymorphic_id_column} = #{resource_table}.id")
+          .where(resource_table => {:id => nil})
+          .where("#{q_table_name}.#{connection.quote_column_name(polymorphic_type_column)} = #{connection.quote(klass.name)}")
       end
-      total
+    end
+
+    def non_polymorphic_orphan_scopes(fk_name, reflection_klass)
+      resource_table = connection.quote_table_name(reflection_klass.table_name)
+      assoc_id = connection.quote_column_name("#{fk_name}_id")
+      q_table_name = connection.quote_table_name(table_name)
+
+      [joins("LEFT OUTER JOIN #{resource_table} ON #{q_table_name}.#{assoc_id} = #{resource_table}.id")
+        .where(resource_table => {:id => nil})]
     end
 
     def polymorphic_classes(polymorphic_type_column)

spec/models/vim_performance_state/purging_spec.rb

@@ -1,26 +1,34 @@
 RSpec.describe VimPerformanceState do
   context "::Purging" do
     describe ".purge_by_orphaned" do
-      it "purges all the orphaned rows for all referenced classes" do
+      before do
         # Create an orphaned row referencing VmOrTemplate
         vm1      = FactoryBot.create(:vm_or_template)
-        vm_perf1 = FactoryBot.create(:vim_performance_state, :resource => vm1)
+        @vm_perf1 = FactoryBot.create(:vim_performance_state, :resource => vm1)
         vm1.delete
 
         # Create a non-orphaned row referencing VmOrTemplate
-        vm_perf2 = FactoryBot.create(:vim_performance_state, :resource => FactoryBot.create(:vm_or_template))
+        @vm_perf2 = FactoryBot.create(:vim_performance_state, :resource => FactoryBot.create(:vm_or_template))
 
         # Create an orphaned row referencing ExtManagementSystem
         ems1      = FactoryBot.create(:ext_management_system)
-        ems_perf1 = FactoryBot.create(:vim_performance_state, :resource => ems1)
+        @ems_perf1 = FactoryBot.create(:vim_performance_state, :resource => ems1)
         ems1.delete
 
         # Create a non-orphaned row referencing ExtManagementSystem
-        ems_perf2 = FactoryBot.create(:vim_performance_state, :resource => FactoryBot.create(:ext_management_system))
+        @ems_perf2 = FactoryBot.create(:vim_performance_state, :resource => FactoryBot.create(:ext_management_system))
+        expect(described_class.all).to match_array([@vm_perf1, @vm_perf2, @ems_perf1, @ems_perf2])
+      end
 
-        expect(described_class.all).to match_array([vm_perf1, vm_perf2, ems_perf1, ems_perf2])
+      it "purges all the orphaned rows for all referenced classes" do
         count = described_class.purge_by_orphaned("resource")
-        expect(described_class.all).to match_array([vm_perf2, ems_perf2])
+        expect(described_class.all).to match_array([@vm_perf2, @ems_perf2])
+        expect(count).to eq(2)
+      end
+
+      it "count purge_mode returns count of the orphaned rows for all referenced classes without actually purging" do
+        count = described_class.purge_by_orphaned("resource", 1000, :count)
+        expect(described_class.all).to match_array([@vm_perf1, @vm_perf2, @ems_perf1, @ems_perf2])
         expect(count).to eq(2)
       end
     end

tools/purge_orphans.rb

@@ -0,0 +1,55 @@
+#!/usr/bin/env ruby
+require "optimist"
+
+VALID_MODES = %w[count purge].freeze
+opts = Optimist.options do
+  synopsis "Purge orphaned rows from several tables."
+  usage "[options]"
+
+  opt :mode, "Mode", :default => "count", :permitted => VALID_MODES
+end
+
+purge_mode = opts[:mode].to_sym
+
+# now load rails
+require File.expand_path('../config/environment', __dir__)
+
+def purge_by_orphaned(klass, fk, window, purge_mode)
+  klass.include PurgingMixin
+  klass.define_method(:purge_method) { :destroy }
+  klass.purge_by_orphaned(fk, window, purge_mode)
+end
+
+CLASSES_TO_PURGE = [
+  BinaryBlob,            :resource,           100,
+  ContainerCondition,    :container_entity,  1000,
+  ContainerEnvVar,       :container,         1000,
+  ContainerPortConfig,   :container,         1000,
+  ContainerVolume,       :parent,            1000,
+  CustomAttribute,       :resource,          1000,
+  MiqReportResultDetail, :miq_report_result, 1000,
+  RequestLog,            :resource,           500,
+  SecurityContext,       :resource,          1000
+]
+
+puts "\nNote: Count mode was provided.  No changes are being made.  Use --mode purge to purge the rows.\n" if purge_mode == :count
+puts
+results = [["Class", purge_mode == :count ? "Orphan Count" : "Purged Rows"]]
+_result, bm = Benchmark.realtime_block("TotalTime") do
+
+  CLASSES_TO_PURGE.each_slice(3) do |klass, fk, window|
+    Benchmark.realtime_block(klass.name) do
+      print "Purging #{klass.name}..."
+      purge_by_orphaned(klass, fk, window, purge_mode).tap { |result| results << [klass, result] }
+      puts "Done!"
+    end
+  end
+  nil
+end
+puts "\nResults:\n\n"
+
+puts results.tableize
+puts
+
+puts "Timing by class:\n\n"
+puts bm.transform_values { |v| v.round(6) }.to_a.unshift(["Model", "Time (s)"]).tableize