1 # All original code copyright 2005, 2006, 2007 Bob Cottrell, Eric Hodel,
2 # The Robot Co-op. All rights reserved.
4 # Redistribution and use in source and binary forms, with or without
5 # modification, are permitted provided that the following conditions
8 # 1. Redistributions of source code must retain the above copyright
9 # notice, this list of conditions and the following disclaimer.
10 # 2. Redistributions in binary form must reproduce the above copyright
11 # notice, this list of conditions and the following disclaimer in the
12 # documentation and/or other materials provided with the distribution.
13 # 3. Neither the names of the authors nor the names of their contributors
14 # may be used to endorse or promote products derived from this software
15 # without specific prior written permission.
17 # THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS
18 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
19 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 # ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE
21 # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
22 # OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT
23 # OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
24 # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
25 # WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
26 # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
27 # EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
38 # Uses the ITU-T polynomial in the CRC32 algorithm.
48 r
= (r
>>1) ^
0xEDB88320
61 # A Ruby client library for memcached.
63 # This is intended to provide access to basic memcached functionality. It
64 # does not attempt to be complete implementation of the entire API, but it is
65 # approaching a complete implementation.
70 # The version of MemCache you are using.
75 # Default options for the cache object.
80 :multithread => false,
84 # Default memcached port.
89 # Default memcached server weight.
94 # The amount of time to wait for a response from a memcached server. If a
95 # response is not completed within this time, the connection to the server
96 # will be closed and an error will be raised.
98 attr_accessor
:request_timeout
101 # The namespace for this instance
103 attr_reader
:namespace
106 # The multithread setting for this instance
108 attr_reader
:multithread
111 # The servers this client talks to. Play at your own peril.
116 # Accepts a list of +servers+ and a list of +opts+. +servers+ may be
117 # omitted. See +servers=+ for acceptable server list arguments.
119 # Valid options for +opts+ are:
121 # [:namespace] Prepends this value to all keys added or retrieved.
122 # [:readonly] Raises an exception on cache writes when true.
123 # [:multithread] Wraps cache access in a Mutex for thread safety.
125 # Other options are ignored.
127 def initialize(*args
)
136 when Hash
then opts
= arg
137 when Array
then servers
= arg
138 when String
then servers
= [arg
]
139 else raise ArgumentError
, 'first argument must be Array, Hash or String'
144 raise ArgumentError
, "wrong number of arguments (#{args.length} for 2)"
147 opts
= DEFAULT_OPTIONS
.merge opts
148 @namespace = opts
[:namespace]
149 @readonly = opts
[:readonly]
150 @multithread = opts
[:multithread]
151 @mutex = Mutex
.new
if @multithread
153 self.servers
= servers
157 # Returns a string representation of the cache object.
160 "<MemCache: %d servers, %d buckets, ns: %p, ro: %p>" %
161 [@servers.length
, @buckets.length
, @namespace, @readonly]
165 # Returns whether there is at least one active server for the object.
172 # Returns whether or not the cache object was created read only.
179 # Set the servers that the requests will be distributed between. Entries
180 # can be either strings of the form "hostname:port" or
181 # "hostname:port:weight" or MemCache::Server objects.
183 def servers
=(servers
)
184 # Create the server objects.
185 @servers = servers
.collect
do |server
|
188 host
, port
, weight
= server
.split
':', 3
189 port
||= DEFAULT_PORT
190 weight
||= DEFAULT_WEIGHT
191 Server
.new
self, host
, port
, weight
193 if server
.memcache
.multithread
!= @multithread then
194 raise ArgumentError
, "can't mix threaded and non-threaded servers"
198 raise TypeError
, "cannot convert #{server.class} into MemCache::Server"
202 # Create an array of server buckets for weight selection of servers.
204 @servers.each
do |server
|
205 server
.weight
.times
{ @buckets.push(server
) }
210 # Decrements the value for +key+ by +amount+ and returns the new value.
211 # +key+ must already exist. If +key+ is not an integer, it is assumed to be
212 # 0. +key+ can not be decremented below 0.
214 def decr(key
, amount
= 1)
215 server
, cache_key
= request_setup key
218 threadsafe_cache_decr server
, cache_key
, amount
220 cache_decr server
, cache_key
, amount
222 rescue TypeError
, SocketError
, SystemCallError
, IOError
=> err
223 handle_error server
, err
227 # Retrieves +key+ from memcache. If +raw+ is false, the value will be
230 def get(key
, raw
= false)
231 server
, cache_key
= request_setup key
233 value
= if @multithread then
234 threadsafe_cache_get server
, cache_key
236 cache_get server
, cache_key
239 return nil if value
.nil?
241 value
= Marshal
.load value
unless raw
244 rescue TypeError
, SocketError
, SystemCallError
, IOError
=> err
245 handle_error server
, err
249 # Retrieves multiple values from memcached in parallel, if possible.
251 # The memcached protocol supports the ability to retrieve multiple
252 # keys in a single request. Pass in an array of keys to this method
255 # 1. map the key to the appropriate memcached server
256 # 2. send a single request to each server that has one or more key values
258 # Returns a hash of values.
262 # cache.get_multi "a", "b" # => { "a" => 1, "b" => 2 }
265 raise MemCacheError
, 'No active servers' unless active
?
268 key_count
= keys
.length
270 server_keys
= Hash
.new
{ |h
,k
| h
[k
] = [] }
272 # map keys to servers
274 server
, cache_key
= request_setup key
275 cache_keys
[cache_key
] = key
276 server_keys
[server
] << cache_key
281 server_keys
.each
do |server
, keys_for_server
|
282 keys_for_server
= keys_for_server
.join
' '
283 values
= if @multithread then
284 threadsafe_cache_get_multi server
, keys_for_server
286 cache_get_multi server
, keys_for_server
288 values
.each
do |key
, value
|
289 results
[cache_keys
[key
]] = Marshal
.load value
294 rescue TypeError
, SocketError
, SystemCallError
, IOError
=> err
295 handle_error server
, err
299 # Increments the value for +key+ by +amount+ and retruns the new value.
300 # +key+ must already exist. If +key+ is not an integer, it is assumed to be
303 def incr(key
, amount
= 1)
304 server
, cache_key
= request_setup key
307 threadsafe_cache_incr server
, cache_key
, amount
309 cache_incr server
, cache_key
, amount
311 rescue TypeError
, SocketError
, SystemCallError
, IOError
=> err
312 handle_error server
, err
316 # Add +key+ to the cache with value +value+ that expires in +expiry+
317 # seconds. If +raw+ is true, +value+ will not be Marshalled.
319 # Warning: Readers should not call this method in the event of a cache miss;
322 def set(key
, value
, expiry
= 0, raw
= false)
323 raise MemCacheError
, "Update of readonly cache" if @readonly
324 server
, cache_key
= request_setup key
325 socket
= server
.socket
327 value
= Marshal
.dump value
unless raw
328 command
= "set #{cache_key} 0 #{expiry} #{value.size}\r\n#{value}\r\n"
331 @mutex.lock
if @multithread
334 raise_on_error_response
! result
336 rescue SocketError
, SystemCallError
, IOError
=> err
338 raise MemCacheError
, err
.message
340 @mutex.unlock
if @multithread
345 # Add +key+ to the cache with value +value+ that expires in +expiry+
346 # seconds, but only if +key+ does not already exist in the cache.
347 # If +raw+ is true, +value+ will not be Marshalled.
349 # Readers should call this method in the event of a cache miss, not
350 # MemCache#set or MemCache#[]=.
352 def add(key
, value
, expiry
= 0, raw
= false)
353 raise MemCacheError
, "Update of readonly cache" if @readonly
354 server
, cache_key
= request_setup key
355 socket
= server
.socket
357 value
= Marshal
.dump value
unless raw
358 command
= "add #{cache_key} 0 #{expiry} #{value.size}\r\n#{value}\r\n"
361 @mutex.lock
if @multithread
364 raise_on_error_response
! result
366 rescue SocketError
, SystemCallError
, IOError
=> err
368 raise MemCacheError
, err
.message
370 @mutex.unlock
if @multithread
375 # Removes +key+ from the cache in +expiry+ seconds.
377 def delete(key
, expiry
= 0)
378 @mutex.lock
if @multithread
380 raise MemCacheError
, "No active servers" unless active
?
381 cache_key
= make_cache_key key
382 server
= get_server_for_key cache_key
385 raise MemCacheError
, "No connection to server" if sock
.nil?
388 sock
.write
"delete #{cache_key} #{expiry}\r\n"
390 raise_on_error_response
! result
392 rescue SocketError
, SystemCallError
, IOError
=> err
394 raise MemCacheError
, err
.message
397 @mutex.unlock
if @multithread
401 # Flush the cache from all memcache servers.
404 raise MemCacheError
, 'No active servers' unless active
?
405 raise MemCacheError
, "Update of readonly cache" if @readonly
407 @mutex.lock
if @multithread
408 @servers.each
do |server
|
411 raise MemCacheError
, "No connection to server" if sock
.nil?
412 sock
.write
"flush_all\r\n"
414 raise_on_error_response
! result
416 rescue SocketError
, SystemCallError
, IOError
=> err
418 raise MemCacheError
, err
.message
422 @mutex.unlock
if @multithread
427 # Reset the connection to all memcache servers. This should be called if
428 # there is a problem with a cache lookup that might have left the connection
429 # in a corrupted state.
432 @servers.each
{ |server
| server
.close
}
436 # Returns statistics for each memcached server. An explanation of the
437 # statistics can be found in the memcached docs:
439 # http://code.sixapart.com/svn/memcached/trunk/server/doc/protocol.txt
444 # {"localhost:11211"=>
447 # "connection_structures"=>4,
448 # "time"=>1162278121,
449 # "pointer_size"=>32,
450 # "limit_maxbytes"=>67108864,
452 # "version"=>"1.2.0",
453 # "bytes_written"=>432583,
456 # "total_connections"=>19,
457 # "curr_connections"=>3,
462 # "rusage_system"=>0.313952,
463 # "rusage_user"=>0.119981,
464 # "bytes_read"=>190619}}
468 raise MemCacheError
, "No active servers" unless active
?
471 @servers.each
do |server
|
473 raise MemCacheError
, "No connection to server" if sock
.nil?
477 sock
.write
"stats\r\n"
479 while line
= sock
.gets
do
480 raise_on_error_response
! line
481 break if line
== "END\r\n"
482 if line
=~
/\ASTAT ([\w]+) ([\w\.\:]+)/ then
484 stats
[name
] = case name
487 when 'rusage_user', 'rusage_system' then
488 seconds
, microseconds
= value
.split(/:/, 2)
490 Float(seconds
) + (Float(microseconds
) / 1_000_000)
492 if value
=~
/\A\d+\Z/ then
500 server_stats
["#{server.host}:#{server.port}"] = stats
501 rescue SocketError
, SystemCallError
, IOError
=> err
503 raise MemCacheError
, err
.message
511 # Shortcut to get a value from the cache.
516 # Shortcut to save a value in the cache. This method does not set an
517 # expiration on the entry. Use set to specify an explicit expiry.
526 # Create a key for the cache, incorporating the namespace qualifier if
529 def make_cache_key(key
)
530 if namespace
.nil? then
533 "#{@namespace}:#{key}"
538 # Pick a server to handle the request based on a hash of the key.
540 def get_server_for_key(key
)
541 raise ArgumentError
, "illegal character in key #{key.inspect}" if
543 raise ArgumentError
, "key too long #{key.inspect}" if key
.length
> 250
544 raise MemCacheError
, "No servers available" if @servers.empty
?
545 return @servers.first
if @servers.length
== 1
550 server
= @buckets[hkey
% @buckets.nitems
]
551 return server
if server
.alive
?
552 hkey
+= hash_for
"#{try}#{key}"
555 raise MemCacheError
, "No servers available"
559 # Returns an interoperable hash value for +key+. (I think, docs are
560 # sketchy for down servers).
563 (key
.crc32_ITU_T
>> 16) & 0x7fff
567 # Performs a raw decr for +cache_key+ from +server+. Returns nil if not
570 def cache_decr(server
, cache_key
, amount
)
571 socket
= server
.socket
572 socket
.write
"decr #{cache_key} #{amount}\r\n"
574 raise_on_error_response
! text
575 return nil if text
== "NOT_FOUND\r\n"
580 # Fetches the raw data for +cache_key+ from +server+. Returns nil on cache
583 def cache_get(server
, cache_key
)
584 socket
= server
.socket
585 socket
.write
"get #{cache_key}\r\n"
586 keyline
= socket
.gets
# "VALUE <key> <flags> <bytes>\r\n"
590 raise MemCacheError
, "lost connection to #{server.host}:#{server.port}"
593 raise_on_error_response
! keyline
594 return nil if keyline
== "END\r\n"
596 unless keyline
=~
/(\d+)\r/ then
598 raise MemCacheError
, "unexpected response #{keyline.inspect}"
600 value
= socket
.read
$1.to_i
601 socket
.read
2 # "\r\n"
602 socket
.gets
# "END\r\n"
607 # Fetches +cache_keys+ from +server+ using a multi-get.
609 def cache_get_multi(server
, cache_keys
)
611 socket
= server
.socket
612 socket
.write
"get #{cache_keys}\r\n"
614 while keyline
= socket
.gets
do
615 return values
if keyline
== "END\r\n"
616 raise_on_error_response
! keyline
618 unless keyline
=~
/\AVALUE (.+) (.+) (.+)/ then
620 raise MemCacheError
, "unexpected response #{keyline.inspect}"
623 key
, data_length
= $1, $3
624 values
[$1] = socket
.read data_length
.to_i
625 socket
.read(2) # "\r\n"
629 raise MemCacheError
, "lost connection to #{server.host}:#{server.port}"
633 # Performs a raw incr for +cache_key+ from +server+. Returns nil if not
636 def cache_incr(server
, cache_key
, amount
)
637 socket
= server
.socket
638 socket
.write
"incr #{cache_key} #{amount}\r\n"
640 raise_on_error_response
! text
641 return nil if text
== "NOT_FOUND\r\n"
646 # Handles +error+ from +server+.
648 def handle_error(server
, error
)
649 server
.close
if server
650 new_error
= MemCacheError
.new error
.message
651 new_error
.set_backtrace error
.backtrace
656 # Performs setup for making a request with +key+ from memcached. Returns
657 # the server to fetch the key from and the complete key to use.
659 def request_setup(key
)
660 raise MemCacheError
, 'No active servers' unless active
?
661 cache_key
= make_cache_key key
662 server
= get_server_for_key cache_key
663 raise MemCacheError
, 'No connection to server' if server
.socket
.nil?
664 return server
, cache_key
667 def threadsafe_cache_decr(server
, cache_key
, amount
) # :nodoc:
669 cache_decr server
, cache_key
, amount
674 def threadsafe_cache_get(server
, cache_key
) # :nodoc:
676 cache_get server
, cache_key
681 def threadsafe_cache_get_multi(socket
, cache_keys
) # :nodoc:
683 cache_get_multi socket
, cache_keys
688 def threadsafe_cache_incr(server
, cache_key
, amount
) # :nodoc:
690 cache_incr server
, cache_key
, amount
695 def raise_on_error_response
!(response
)
696 if response
=~
/\A(?:CLIENT_|SERVER_)?ERROR (.*)/
697 raise MemCacheError
, $1.strip
703 # This class represents a memcached server instance.
708 # The amount of time to wait to establish a connection with a memcached
709 # server. If a connection cannot be established within this time limit,
710 # the server will be marked as down.
712 CONNECT_TIMEOUT
= 0.25
715 # The amount of time to wait before attempting to re-establish a
716 # connection with a server that is marked dead.
721 # The host the memcached server is running on.
726 # The port the memcached server is listening on.
731 # The weight given to the server.
736 # The time of next retry if the connection is dead.
741 # A text status string describing the state of the server.
746 # Create a new MemCache::Server object for the memcached instance
747 # listening on the given host and port, weighted by the given weight.
749 def initialize(memcache
, host
, port
= DEFAULT_PORT
, weight
= DEFAULT_WEIGHT
)
750 raise ArgumentError
, "No host specified" if host
.nil? or host
.empty
?
751 raise ArgumentError
, "No port specified" if port
.nil? or port
.to_i
.zero
?
756 @weight = weight
.to_i
758 @multithread = @memcache.multithread
763 @status = 'NOT CONNECTED'
767 # Return a string representation of the server object.
770 "<MemCache::Server: %s:%d [%d] (%s)>" % [@host, @port, @weight, @status]
774 # Check whether the server connection is alive. This will cause the
775 # socket to attempt to connect if it isn't already connected and or if
776 # the server was previously marked as down and the retry time has
784 # Try to connect to the memcached server targeted by this object.
785 # Returns the connected socket object on success or nil on failure.
788 @mutex.lock
if @multithread
789 return @sock if @sock and not @sock.closed
?
793 # If the host was dead, don't retry for a while.
794 return if @retry and @retry > Time
.now
796 # Attempt to connect if not already connected.
798 @sock = timeout CONNECT_TIMEOUT
do
799 TCPSocket
.new
@host, @port
801 if Socket
.constants
.include? 'TCP_NODELAY' then
802 @sock.setsockopt Socket
::IPPROTO_TCP, Socket
::TCP_NODELAY, 1
805 @status = 'CONNECTED'
806 rescue SocketError
, SystemCallError
, IOError
, Timeout
::Error => err
807 mark_dead err
.message
812 @mutex.unlock
if @multithread
816 # Close the connection to the memcached server targeted by this
817 # object. The server is not considered dead.
820 @mutex.lock
if @multithread
821 @sock.close
if @sock && !@sock.closed
?
824 @status = "NOT CONNECTED"
826 @mutex.unlock
if @multithread
832 # Mark the server as dead and close its socket.
834 def mark_dead(reason
= "Unknown error")
835 @sock.close
if @sock && !@sock.closed
?
837 @retry = Time
.now
+ RETRY_DELAY
839 @status = sprintf
"DEAD: %s, will retry at %s", reason
, @retry
844 # Base MemCache exception class.
846 class MemCacheError
< RuntimeError
; end