vendor/rails/activerecord/lib/active_record/batches.rb

   1 module ActiveRecord
   2   module Batches # :nodoc:
   3     def self.included(base)
   4       base.extend(ClassMethods)
   5     end
   6
   7     # When processing large numbers of records, it's often a good idea to do so in batches to prevent memory ballooning.
   8     module ClassMethods
   9       # Yields each record that was found by the find +options+. The find is performed by find_in_batches
  10       # with a batch size of 1000 (or as specified by the +batch_size+ option).
  11       #
  12       # Example:
  13       #
  14       #   Person.find_each(:conditions => "age > 21") do |person|
  15       #     person.party_all_night!
  16       #   end
  17       #
  18       # Note: This method is only intended to use for batch processing of large amounts of records that wouldn't fit in
  19       # memory all at once. If you just need to loop over less than 1000 records, it's probably better just to use the
  20       # regular find methods.
  21       def find_each(options = {})
  22         find_in_batches(options) do |records|
  23           records.each { |record| yield record }
  24         end
  25
  26         self
  27       end
  28
  29       # Yields each batch of records that was found by the find +options+ as an array. The size of each batch is
  30       # set by the +batch_size+ option; the default is 1000.
  31       #
  32       # You can control the starting point for the batch processing by supplying the +start+ option. This is especially
  33       # useful if you want multiple workers dealing with the same processing queue. You can make worker 1 handle all the
  34       # records between id 0 and 10,000 and worker 2 handle from 10,000 and beyond (by setting the +start+ option on that
  35       # worker).
  36       #
  37       # It's not possible to set the order. That is automatically set to ascending on the primary key ("id ASC")
  38       # to make the batch ordering work. This also mean that this method only works with integer-based primary keys.
  39       # You can't set the limit either, that's used to control the the batch sizes.
  40       #
  41       # Example:
  42       #
  43       #   Person.find_in_batches(:conditions => "age > 21") do |group|
  44       #     sleep(50) # Make sure it doesn't get too crowded in there!
  45       #     group.each { |person| person.party_all_night! }
  46       #   end
  47       def find_in_batches(options = {})
  48         raise "You can't specify an order, it's forced to be #{batch_order}" if options[:order]
  49         raise "You can't specify a limit, it's forced to be the batch_size"  if options[:limit]
  50
  51         start = options.delete(:start).to_i
  52         batch_size = options.delete(:batch_size) || 1000
  53
  54         with_scope(:find => options.merge(:order => batch_order, :limit => batch_size)) do
  55           records = find(:all, :conditions => [ "#{table_name}.#{primary_key} >= ?", start ])
  56
  57           while records.any?
  58             yield records
  59
  60             break if records.size < batch_size
  61             records = find(:all, :conditions => [ "#{table_name}.#{primary_key} > ?", records.last.id ])
  62           end
  63         end
  64       end
  65
  66
  67       private
  68         def batch_order
  69           "#{table_name}.#{primary_key} ASC"
  70         end
  71     end
  72   end
  73 end