2 module Batches
# :nodoc:
3 def self.included(base
)
4 base
.extend(ClassMethods
)
7 # When processing large numbers of records, it's often a good idea to do so in batches to prevent memory ballooning.
9 # Yields each record that was found by the find +options+. The find is performed by find_in_batches
10 # with a batch size of 1000 (or as specified by the +batch_size+ option).
14 # Person.find_each(:conditions => "age > 21") do |person|
15 # person.party_all_night!
18 # Note: This method is only intended to use for batch processing of large amounts of records that wouldn't fit in
19 # memory all at once. If you just need to loop over less than 1000 records, it's probably better just to use the
20 # regular find methods.
21 def find_each(options
= {})
22 find_in_batches(options
) do |records
|
23 records
.each
{ |record
| yield record
}
29 # Yields each batch of records that was found by the find +options+ as an array. The size of each batch is
30 # set by the +batch_size+ option; the default is 1000.
32 # You can control the starting point for the batch processing by supplying the +start+ option. This is especially
33 # useful if you want multiple workers dealing with the same processing queue. You can make worker 1 handle all the
34 # records between id 0 and 10,000 and worker 2 handle from 10,000 and beyond (by setting the +start+ option on that
37 # It's not possible to set the order. That is automatically set to ascending on the primary key ("id ASC")
38 # to make the batch ordering work. This also mean that this method only works with integer-based primary keys.
39 # You can't set the limit either, that's used to control the the batch sizes.
43 # Person.find_in_batches(:conditions => "age > 21") do |group|
44 # sleep(50) # Make sure it doesn't get too crowded in there!
45 # group.each { |person| person.party_all_night! }
47 def find_in_batches(options
= {})
48 raise "You can't specify an order, it's forced to be #{batch_order}" if options
[:order]
49 raise "You can't specify a limit, it's forced to be the batch_size" if options
[:limit]
51 start
= options
.delete(:start).to_i
52 batch_size
= options
.delete(:batch_size) || 1000
54 with_scope(:find => options
.merge(:order => batch_order
, :limit => batch_size
)) do
55 records
= find(:all, :conditions => [ "#{table_name}.#{primary_key} >= ?", start
])
60 break if records
.size
< batch_size
61 records
= find(:all, :conditions => [ "#{table_name}.#{primary_key} > ?", records
.last
.id
])
69 "#{table_name}.#{primary_key} ASC"