From fa220d14dc97129c7545ce3132d1d09a7321b94e Mon Sep 17 00:00:00 2001
From: BJ Neilsen <bj.neilsen@gmail.com>
Date: Tue, 19 Feb 2013 15:31:11 -0700
Subject: [PATCH 01/70] First cut of the Lazy Pirate client pattern

The ZMQ client connector implements the Lazy Pirate pattern as laid out
in the [ZMQ Guide - Chapter 4][lpp]. The essence is that we attempt to
be more stable in our client requests as we assume the server may not be
available for fulfilling responses.

A connection will be created to the server and data sent with a timeout
on the socket. If the socket becomes readable inside the timeout, we
read the data and go on our way.

If the socket does not become readable in the timeout, we will close and
reopen the socket, send the data again, and again wait for a valid
response or timeout.

If we exceed the number of tries we do not attempt the connection
anymore and we fail with a timeout message to the calling code.

  [lpp]:
  http://zguide.zeromq.org/php:chapter4#Client-side-Reliability-Lazy-Pirate-Pattern
---
 lib/protobuf/rpc/connectors/zmq.rb     | 154 ++++++++++++++++++++-----
 lib/protobuf/rpc/servers/zmq/broker.rb |   2 +-
 2 files changed, 127 insertions(+), 29 deletions(-)

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index 993ac3e1..0e5fcceb 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -4,20 +4,35 @@ module Protobuf
   module Rpc
     module Connectors
       class Zmq < Base
+
+        ##
+        # Included Modules
+        #
+
         include Protobuf::Rpc::Connectors::Common
         include Protobuf::Logger::LogMethods
 
+        ##
+        # Class Constants
+        #
+
+        LAZY_RETRIES = 3
+
+        ##
+        # Instance methods
+        #
+
+        # Start the request/response cycle. We implement the Lazy Pirate
+        # req/reply reliability pattern as laid out in the ZMQ Guide, Chapter 4.
+        #
+        # @see http://zguide.zeromq.org/php:chapter4#Client-side-Reliability-Lazy-Pirate-Pattern
+        #
         def send_request
-          timeout_wrap do
-            setup_connection
-            connect_to_rpc_server
-            post_init
-            read_response
-          end
+          setup_connection
+          connect_to_rpc_server
+          poll_send_data
         ensure
-          @socket.close if @socket
-          @zmq_context.terminate if @zmq_context
-          @zmq_context = nil
+          close_connection
         end
 
         def log_signature
@@ -26,45 +41,128 @@ def log_signature
 
         private
 
+        ##
+        # Private Instance methods
+        #
+
         def close_connection
-          return if @error
-          zmq_error_check(@socket.close)
-          zmq_error_check(@zmq_context.terminate)
-          log_debug { sign_message("Connector closed")  }
+          socket_close
+          zmq_context_terminate
         end
 
+        # Establish a request socket connection to the remote rpc_server.
+        # Set the socket option LINGER to 0 so that we don't wait
+        # for queued messages to be accepted when the socket/context are
+        # asked to close/terminate.
+        #
         def connect_to_rpc_server
-          return if @error
-          log_debug { sign_message("Establishing connection: #{options[:host]}:#{options[:port]}") }
-          @zmq_context = ::ZMQ::Context.new
-          @socket = @zmq_context.socket(::ZMQ::REQ)
-          zmq_error_check(@socket.connect("tcp://#{options[:host]}:#{options[:port]}"))
-          log_debug { sign_message("Connection established #{options[:host]}:#{options[:port]}") }
+          return if error?
+
+          location = "#{options[:host]}:#{options[:port]}"
+          log_debug { sign_message("Establishing connection: #{location}") }
+          socket.setsockopt(::ZMQ::LINGER, 0)
+          zmq_error_check(socket.connect("tcp://#{location}"), :socket_connect)
+          zmq_error_check(poller.register_readable(socket), :poller_register_socket)
+          log_debug { sign_message("Connection established to #{location}") }
         end
 
-        # Method to determine error state, must be used with Connector api
+        # Method to determine error state, must be used with Connector API.
+        #
         def error?
-          !!@error
+          !! @error
+        end
+
+        # Trying a number of times, attempt to get a response from the server.
+        # If we haven't received a legitimate response in the LAZY_RETRIES number
+        # of retries, fail the request.
+        def poll_send_data
+          poll_timeout = (options[:timeout] / LAZY_RETRIES) * 1000
+
+          LAZY_RETRIES.times do |n|
+            send_data
+
+            if poller.poll(poll_timeout) == 1
+              read_response
+              return
+            else
+              socket_close
+              connect_to_rpc_server
+            end
+          end
+
+          fail(:RPC_FAILED, "The server took longer than #{options[:timeout]} seconds to respond")
+        end
+
+        def poller
+          @poller ||= ::ZMQ::Poller.new
         end
 
+        # Read the string response from the available readable. This will be
+        # the current @socket. Calls `parse_response` to invoke the success or
+        # failed callbacks, depending on the state of the communication
+        # and response data.
+        #
         def read_response
-          return if @error
-          @response_data = ''
-          zmq_error_check(@socket.recv_string(@response_data))
+          return if error?
+
+          poller.readables.each do |readable|
+            @response_data = ''
+            zmq_error_check(readable.recv_string(@response_data), :socket_recv_string)
+          end
+
           parse_response
         end
 
+        # Send the request data to the remote rpc_server.
+        #
         def send_data
-          return if @error
           log_debug { sign_message("Sending Request: #{@request_data}") }
           @stats.request_size = @request_data.size
-          zmq_error_check(@socket.send_string(@request_data))
+          zmq_error_check(socket.send_string(@request_data), :socket_send_string)
           log_debug { sign_message("write closed") }
         end
 
-        def zmq_error_check(return_code)
-          raise "Last API call failed at #{caller(1)}" unless return_code >= 0
+        # Setup a ZMQ request socket in the current zmq context.
+        #
+        def socket
+          @socket ||= zmq_context.socket(::ZMQ::REQ)
+        end
+
+        def socket_close
+          if socket
+            log_debug { sign_message("Closing Socket")  }
+            zmq_error_check(poller.deregister_readable(socket), :poller_deregister_socket)
+            zmq_error_check(socket.close, :socket_close)
+            log_debug { sign_message("Socket closed")  }
+            @socket = nil
+          end
+        end
+
+        def zmq_context
+          self.class.zmq_context
         end
+
+        # Return the ZMQ Context to use for this process.
+        # If the context does not exist, create it, then register
+        # an exit block to ensure the context is terminated correctly.
+        #
+        def zmq_context
+          @zmq_context ||= ::ZMQ::Context.new
+        end
+
+        # Terminate the zmq_context (if any).
+        #
+        def zmq_context_terminate
+          log_debug { sign_message("Terminating ZMQ Context")  }
+          @zmq_context.try(:terminate)
+          @zmq_context = nil
+          log_debug { sign_message("ZMQ Context terminated")  }
+        end
+
+        def zmq_error_check(return_code, message)
+          raise "Last ZMQ API call (#{message}) failed at #{caller(1)}" unless return_code >= 0
+        end
+
       end
     end
   end
diff --git a/lib/protobuf/rpc/servers/zmq/broker.rb b/lib/protobuf/rpc/servers/zmq/broker.rb
index 01de8c8f..f048ef87 100644
--- a/lib/protobuf/rpc/servers/zmq/broker.rb
+++ b/lib/protobuf/rpc/servers/zmq/broker.rb
@@ -28,7 +28,7 @@ def poll
           if available_workers.size > 0
             poller.register(frontend, ::ZMQ::POLLIN) if poller.size < 2
           else
-            poller.delete(frontend)            
+            poller.delete(frontend)
           end
 
           poller.poll(1000)

From c1fe9ca6a67e5c0bb9d74ec64a8bb842639a7bb5 Mon Sep 17 00:00:00 2001
From: BJ Neilsen <bj.neilsen@gmail.com>
Date: Tue, 19 Feb 2013 15:54:07 -0700
Subject: [PATCH 02/70] Remove overdefined zmq_context method

---
 lib/protobuf/rpc/connectors/zmq.rb | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index 0e5fcceb..553629ff 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -75,6 +75,7 @@ def error?
         # Trying a number of times, attempt to get a response from the server.
         # If we haven't received a legitimate response in the LAZY_RETRIES number
         # of retries, fail the request.
+        #
         def poll_send_data
           poll_timeout = (options[:timeout] / LAZY_RETRIES) * 1000
 
@@ -138,10 +139,6 @@ def socket_close
           end
         end
 
-        def zmq_context
-          self.class.zmq_context
-        end
-
         # Return the ZMQ Context to use for this process.
         # If the context does not exist, create it, then register
         # an exit block to ensure the context is terminated correctly.

From c49ed10cb4a0d3221a3c999e47124de31b77c320 Mon Sep 17 00:00:00 2001
From: BJ Neilsen <bj.neilsen@gmail.com>
Date: Tue, 19 Feb 2013 16:11:04 -0700
Subject: [PATCH 03/70] Add PB_CLIENT_RETRIES env var for client lpp

---
 lib/protobuf/rpc/connectors/zmq.rb | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index 553629ff..246f76b4 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -16,7 +16,7 @@ class Zmq < Base
         # Class Constants
         #
 
-        LAZY_RETRIES = 3
+        CLIENT_RETRIES = ENV['PB_CLIENT_RETRIES'] || 3
 
         ##
         # Instance methods
@@ -62,7 +62,7 @@ def connect_to_rpc_server
           log_debug { sign_message("Establishing connection: #{location}") }
           socket.setsockopt(::ZMQ::LINGER, 0)
           zmq_error_check(socket.connect("tcp://#{location}"), :socket_connect)
-          zmq_error_check(poller.register_readable(socket), :poller_register_socket)
+          zmq_error_check(poller.register_readable(socket), :poller_register_readable)
           log_debug { sign_message("Connection established to #{location}") }
         end
 
@@ -73,13 +73,13 @@ def error?
         end
 
         # Trying a number of times, attempt to get a response from the server.
-        # If we haven't received a legitimate response in the LAZY_RETRIES number
+        # If we haven't received a legitimate response in the CLIENT_RETRIES number
         # of retries, fail the request.
         #
         def poll_send_data
-          poll_timeout = (options[:timeout] / LAZY_RETRIES) * 1000
+          poll_timeout = (options[:timeout] / CLIENT_RETRIES) * 1000
 
-          LAZY_RETRIES.times do |n|
+          CLIENT_RETRIES.times do |n|
             send_data
 
             if poller.poll(poll_timeout) == 1
@@ -132,7 +132,7 @@ def socket
         def socket_close
           if socket
             log_debug { sign_message("Closing Socket")  }
-            zmq_error_check(poller.deregister_readable(socket), :poller_deregister_socket)
+            zmq_error_check(poller.deregister_readable(socket), :poller_deregister_readable)
             zmq_error_check(socket.close, :socket_close)
             log_debug { sign_message("Socket closed")  }
             @socket = nil

From 8f4d3b2928c5ca930081cd643192e3d831234785 Mon Sep 17 00:00:00 2001
From: BJ Neilsen <bj.neilsen@gmail.com>
Date: Tue, 19 Feb 2013 16:35:21 -0700
Subject: [PATCH 04/70] Synchronize zmq_error_check function to include source

---
 lib/protobuf/rpc/connectors/zmq.rb             | 10 ++++++++--
 lib/protobuf/rpc/servers/zmq/broker.rb         | 12 ++++++------
 lib/protobuf/rpc/servers/zmq/util.rb           | 10 ++++++++--
 lib/protobuf/rpc/servers/zmq/worker.rb         | 10 +++++-----
 spec/lib/protobuf/rpc/servers/zmq/util_spec.rb | 10 +++++-----
 5 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index 246f76b4..0040b5d2 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -156,8 +156,14 @@ def zmq_context_terminate
           log_debug { sign_message("ZMQ Context terminated")  }
         end
 
-        def zmq_error_check(return_code, message)
-          raise "Last ZMQ API call (#{message}) failed at #{caller(1)}" unless return_code >= 0
+        def zmq_error_check(return_code, source)
+          unless ::ZMQ::Util.resultcode_ok?(return_code)
+            raise <<-ERROR
+            Last ZMQ API call to #{source} failed with "#{::ZMQ::Util.error_string}".
+
+            #{caller(1).join($/)}
+            ERROR
+          end
         end
 
       end
diff --git a/lib/protobuf/rpc/servers/zmq/broker.rb b/lib/protobuf/rpc/servers/zmq/broker.rb
index f048ef87..905f2722 100644
--- a/lib/protobuf/rpc/servers/zmq/broker.rb
+++ b/lib/protobuf/rpc/servers/zmq/broker.rb
@@ -52,7 +52,7 @@ def teardown
 
           def move_to_backend(socket)
             message_array = []
-            zmq_error_check(socket.recv_strings(message_array))
+            zmq_error_check(socket.recv_strings(message_array), :socket_recv_strings)
 
             backend_message_set = [
               available_workers.shift, # Worker UUID for router
@@ -62,12 +62,12 @@ def move_to_backend(socket)
               message_array[2] # Client Message payload (request)
             ]
 
-            zmq_error_check(backend.send_strings(backend_message_set))
+            zmq_error_check(backend.send_strings(backend_message_set), :backend_send_strings)
           end
 
           def move_to_frontend(socket)
             message_array = []
-            zmq_error_check(socket.recv_strings(message_array))
+            zmq_error_check(socket.recv_strings(message_array), :socked_recv_strings)
 
             # Push UUID of socket on the available workers queue
             available_workers << message_array[0]
@@ -80,7 +80,7 @@ def move_to_frontend(socket)
                 message_array[4] # Reply payload
               ]
 
-              zmq_error_check(frontend.send_strings(frontend_message_set))
+              zmq_error_check(frontend.send_strings(frontend_message_set), :frontend_send_strings)
             end
           end
 
@@ -90,7 +90,7 @@ def setup_backend(options = {})
             port = dealer_options[:port]
 
             zmq_backend = context.socket(::ZMQ::ROUTER)
-            zmq_error_check(zmq_backend.bind(bind_address(host, port)))
+            zmq_error_check(zmq_backend.bind(bind_address(host, port)), :zmq_backend_bind)
             zmq_backend
           end
 
@@ -99,7 +99,7 @@ def setup_frontend(options = {})
             port = options[:port]
 
             zmq_frontend = context.socket(::ZMQ::ROUTER)
-            zmq_error_check(zmq_frontend.bind(bind_address(host, port)))
+            zmq_error_check(zmq_frontend.bind(bind_address(host, port)), :zmq_frontend_bind)
             zmq_frontend
           end
 
diff --git a/lib/protobuf/rpc/servers/zmq/util.rb b/lib/protobuf/rpc/servers/zmq/util.rb
index d8ff4508..2209c409 100644
--- a/lib/protobuf/rpc/servers/zmq/util.rb
+++ b/lib/protobuf/rpc/servers/zmq/util.rb
@@ -10,8 +10,14 @@ def self.included(base)
           base.extend(::Protobuf::Rpc::Zmq::Util)
         end
 
-        def zmq_error_check(return_code)
-          raise "Last API call failed with \"#{::ZMQ::Util.error_string}\"#{$/}#{$/}#{caller(1)}" unless return_code >= 0
+        def zmq_error_check(return_code, source)
+          unless ::ZMQ::Util.resultcode_ok?(return_code)
+            raise <<-ERROR
+            Last ZMQ API call to #{source} failed with "#{::ZMQ::Util.error_string}".
+
+            #{caller(1).join($/)}
+            ERROR
+          end
         end
 
         def log_signature
diff --git a/lib/protobuf/rpc/servers/zmq/worker.rb b/lib/protobuf/rpc/servers/zmq/worker.rb
index 0e969ee4..f3a16204 100644
--- a/lib/protobuf/rpc/servers/zmq/worker.rb
+++ b/lib/protobuf/rpc/servers/zmq/worker.rb
@@ -17,13 +17,13 @@ def initialize(options = {})
 
           @zmq_context = ::ZMQ::Context.new
           @socket = @zmq_context.socket(::ZMQ::REQ)
-          zmq_error_check(@socket.connect("tcp://#{resolve_ip(host)}:#{port}"))
+          zmq_error_check(@socket.connect("tcp://#{resolve_ip(host)}:#{port}"), :socket_connect)
 
           @poller = ::ZMQ::Poller.new
           @poller.register(@socket, ::ZMQ::POLLIN)
 
           # Send request to broker telling it we are ready
-          zmq_error_check(@socket.send_string(::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE))
+          zmq_error_check(@socket.send_string(::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE), :socket_send_string)
         end
 
         ##
@@ -31,7 +31,7 @@ def initialize(options = {})
         #
         def handle_request(socket)
           message_array = []
-          zmq_error_check(socket.recv_strings(message_array))
+          zmq_error_check(socket.recv_strings(message_array), :socket_recv_strings)
 
           @request_data = message_array[2]
           @client_address = message_array[0]
@@ -55,7 +55,7 @@ def run
         end
 
         def send_data
-          response_data = @response.to_s # to_s is aliases as serialize_to_string in Message
+          response_data = @response.serialize_to_string
 
           response_message_set = [
             @client_address, # client uuid address
@@ -64,7 +64,7 @@ def send_data
           ]
 
           @stats.response_size = response_data.size
-          zmq_error_check(@socket.send_strings(response_message_set))
+          zmq_error_check(@socket.send_strings(response_message_set), :socket_send_strings)
         end
       end
 
diff --git a/spec/lib/protobuf/rpc/servers/zmq/util_spec.rb b/spec/lib/protobuf/rpc/servers/zmq/util_spec.rb
index 90a2ef7c..dad46a46 100644
--- a/spec/lib/protobuf/rpc/servers/zmq/util_spec.rb
+++ b/spec/lib/protobuf/rpc/servers/zmq/util_spec.rb
@@ -13,26 +13,26 @@ class UtilTest
   describe '#zmq_error_check' do
     it 'raises when the error code is less than 0' do
       expect {
-        subject.zmq_error_check(-1)
-      }.to raise_error
+        subject.zmq_error_check(-1, :test)
+      }.to raise_error(/test/)
     end
 
     it 'retrieves the error string from ZeroMQ' do
       ZMQ::Util.stub(:error_string).and_return('an error from zmq')
       expect {
-        subject.zmq_error_check(-1)
+        subject.zmq_error_check(-1, :test)
       }.to raise_error(RuntimeError, /an error from zmq/i)
     end
 
     it 'does nothing if the error code is > 0' do
       expect {
-        subject.zmq_error_check(1)
+        subject.zmq_error_check(1, :test)
       }.to_not raise_error
     end
 
     it 'does nothing if the error code is == 0' do
       expect {
-        subject.zmq_error_check(0)
+        subject.zmq_error_check(0, :test)
       }.to_not raise_error
     end
   end

From e6af31970b753d4460d338a7ab8a6334daae445f Mon Sep 17 00:00:00 2001
From: BJ Neilsen <bj.neilsen@gmail.com>
Date: Wed, 20 Feb 2013 14:58:40 -0700
Subject: [PATCH 05/70] Fix the benchmark rake tasks

---
 Rakefile                |  2 +-
 protobuf.gemspec        |  2 +-
 spec/benchmark/tasks.rb | 38 ++++++++++++++++++--------------
 spec/support/server.rb  | 49 +++++++++++++++++++++++++++--------------
 4 files changed, 56 insertions(+), 35 deletions(-)

diff --git a/Rakefile b/Rakefile
index 15f786bc..f41d3f4c 100644
--- a/Rakefile
+++ b/Rakefile
@@ -4,7 +4,7 @@ $:.push File.expand_path("./spec", File.dirname(__FILE__))
 require "rubygems"
 require "rubygems/package_task"
 require "bundler/gem_tasks"
-# require "benchmark/tasks"
+require "benchmark/tasks"
 
 require "rspec/core/rake_task"
 
diff --git a/protobuf.gemspec b/protobuf.gemspec
index d40fbefa..30e1c799 100644
--- a/protobuf.gemspec
+++ b/protobuf.gemspec
@@ -33,7 +33,7 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'eventmachine'
   s.add_development_dependency 'ffi-rzmq'
-  # s.add_development_dependency 'perftools.rb'
+  s.add_development_dependency 'perftools.rb'
   s.add_development_dependency 'pry'
   s.add_development_dependency 'pry-nav'
   s.add_development_dependency 'rake'
diff --git a/spec/benchmark/tasks.rb b/spec/benchmark/tasks.rb
index e573b2ca..726a6ff1 100644
--- a/spec/benchmark/tasks.rb
+++ b/spec/benchmark/tasks.rb
@@ -1,5 +1,6 @@
 require 'benchmark'
-require 'support/all'
+require 'protobuf'
+require 'support/server'
 require 'support/test/resource_service'
 
 begin
@@ -33,11 +34,12 @@ def em_client_em_server(number_tests, test_length, global_bench = nil)
 
     EventMachine.fiber_run do
       StubServer.new do |server|
-        client = ::Test::ResourceService.client
-
         benchmark_wrapper(global_bench) do |bench|
           bench.report("ES / EC") do
-            (1..number_tests.to_i).each { client.find(:name => "Test Name" * test_length.to_i, :active => true) }
+            (1..number_tests.to_i).each do
+              client = ::Test::ResourceService.client
+              client.find(:name => "Test Name" * test_length.to_i, :active => true)
+            end
           end
         end
       end
@@ -51,11 +53,12 @@ def em_client_sock_server(number_tests, test_length, global_bench = nil)
 
     EventMachine.fiber_run do
       StubServer.new(:server => Protobuf::Rpc::Socket::Server, :port => 9399) do |server|
-        client = ::Test::ResourceService.client(:port => 9399)
-
         benchmark_wrapper(global_bench) do |bench|
           bench.report("SS / EC") do
-            (1..number_tests.to_i).each { client.find(:name => "Test Name" * test_length.to_i, :active => true) }
+            (1..number_tests.to_i).each do
+              client = ::Test::ResourceService.client(:port => 9399)
+              client.find(:name => "Test Name" * test_length.to_i, :active => true)
+            end
           end
         end
       end
@@ -69,11 +72,12 @@ def sock_client_sock_server(number_tests, test_length, global_bench = nil)
     EM.stop if EM.reactor_running?
 
     StubServer.new(:server => Protobuf::Rpc::Socket::Server, :port => 9399) do |server|
-      client = ::Test::ResourceService.client(:port => 9399)
-
       benchmark_wrapper(global_bench) do |bench|
         bench.report("SS / SC") do
-          (1..number_tests.to_i).each { client.find(:name => "Test Name" * test_length.to_i, :active => true) }
+          (1..number_tests.to_i).each do
+            client = ::Test::ResourceService.client(:port => 9399)
+            client.find(:name => "Test Name" * test_length.to_i, :active => true)
+          end
         end
       end
     end
@@ -86,11 +90,12 @@ def sock_client_em_server(number_tests, test_length, global_bench = nil)
     Thread.pass until EM.reactor_running?
 
     StubServer.new(:port => 9399) do |server|
-      client = ::Test::ResourceService.client(:port => 9399)
-
       benchmark_wrapper(global_bench) do |bench|
         bench.report("ES / SC") do
-          (1..number_tests.to_i).each { client.find(:name => "Test Name" * test_length.to_i, :active => true) }
+          (1..number_tests.to_i).each do
+            client = ::Test::ResourceService.client(:port => 9399)
+            client.find(:name => "Test Name" * test_length.to_i, :active => true)
+          end
         end
       end
     end
@@ -102,11 +107,12 @@ def sock_client_em_server(number_tests, test_length, global_bench = nil)
   def zmq_client_zmq_server(number_tests, test_length, global_bench = nil)
     load "protobuf/zmq.rb"
     StubServer.new(:port => 9399, :server => Protobuf::Rpc::Zmq::Server) do |server|
-      client = ::Test::ResourceService.client(:port => 9399)
-
       benchmark_wrapper(global_bench) do |bench|
         bench.report("ZS / ZC") do
-          (1..number_tests.to_i).each { client.find(:name => "Test Name" * test_length.to_i, :active => true) }
+          (1..number_tests.to_i).each do |i|
+            client = ::Test::ResourceService.client(:port => 9399)
+            client.find(:name => "Test Name" * test_length.to_i, :active => true)
+          end
         end
       end
       server.stop
diff --git a/spec/support/server.rb b/spec/support/server.rb
index 4530d14f..0982258b 100644
--- a/spec/support/server.rb
+++ b/spec/support/server.rb
@@ -41,19 +41,28 @@ def initialize(options = {})
     @running = true
     @options = OpenStruct.new({ :host => "127.0.0.1",
                                 :port => 9399,
-                                :delay => 0,
-                                :server => Protobuf::Rpc::Evented::Server }.merge(options))
+                                :backlog => 100,
+                                :threshold => 100,
+                                :threads => 5,
+                                :server => Protobuf::Rpc::Zmq::Server }.merge(options))
 
     start
     yield self
+  rescue => e
+    puts <<-EXC
+    Exception occurred #{e.message}
+    #{e.backtrace.join($/)}
+    EXC
   ensure
     stop if @running
   end
 
   def start
-    case
-    when @options.server == Protobuf::Rpc::Evented::Server then start_em_server
-    when @options.server == Protobuf::Rpc::Zmq::Server then start_zmq_server
+    case @options.server.name
+    when "Protobuf::Rpc::Evented::Server" then
+      start_em_server
+    when "Protobuf::Rpc::Zmq::Server" then
+      start_zmq_server
     else
       start_socket_server
     end
@@ -65,26 +74,32 @@ def start
     end
   end
 
-  def start_em_server
-    @server_handle = EventMachine.start_server(@options.host, @options.port, StubProtobufServerFactory.build(@options.delay))
+  if defined?(EventMachine)
+    def start_em_server
+      @server_handle = EventMachine.start_server(@options.host, @options.port, StubProtobufServerFactory.build(@options.delay))
+    end
   end
 
-  def start_socket_server
-    @sock_server = Thread.new(@options) { |opt| Protobuf::Rpc::SocketRunner.run(opt) }
-    @sock_server.abort_on_exception = true # Set for testing purposes
-    Thread.pass until Protobuf::Rpc::Socket::Server.running?
+  if defined?(Protobuf::Rpc::Socket::Server)
+    def start_socket_server
+      @sock_server = Thread.new(@options) { |opt| Protobuf::Rpc::SocketRunner.run(opt) }
+      @sock_server.abort_on_exception = true # Set for testing purposes
+      Thread.pass until Protobuf::Rpc::Socket::Server.running?
+    end
   end
 
-  def start_zmq_server
-    @zmq_server = Thread.new(@options) { |opt| Protobuf::Rpc::ZmqRunner.run(opt) }
-    Thread.pass until Protobuf::Rpc::Zmq::Server.running?
+  if defined?(Protobuf::Rpc::Zmq::Server)
+    def start_zmq_server
+      @zmq_server = Thread.new(@options) { |opt| Protobuf::Rpc::ZmqRunner.run(opt) }
+      Thread.pass until Protobuf::Rpc::Zmq::Server.running?
+    end
   end
 
   def stop
-    case
-    when @options.server == Protobuf::Rpc::Evented::Server then
+    case @options.server.name
+    when "Protobuf::Rpc::Evented::Server" then
       EventMachine.stop_server(@server_handle) if @server_handle
-    when @options.server == Protobuf::Rpc::Zmq::Server then
+    when "Protobuf::Rpc::Zmq::Server" then
       Protobuf::Rpc::ZmqRunner.stop
       @zmq_server.join if @zmq_server
     else

From f41b640f25a9e950c7d2c025058c8885ca0c3934 Mon Sep 17 00:00:00 2001
From: BJ Neilsen <bj.neilsen@gmail.com>
Date: Wed, 20 Feb 2013 15:03:41 -0700
Subject: [PATCH 06/70] Fix poller timeout on client zmq socket cnxns

---
 lib/protobuf/rpc/connectors/zmq.rb     | 14 ++++++--------
 lib/protobuf/rpc/servers/zmq/broker.rb |  2 +-
 lib/protobuf/rpc/servers/zmq/util.rb   |  7 ++++++-
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index 0040b5d2..34c9488a 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -16,7 +16,7 @@ class Zmq < Base
         # Class Constants
         #
 
-        CLIENT_RETRIES = ENV['PB_CLIENT_RETRIES'] || 3
+        CLIENT_RETRIES = (ENV['PB_CLIENT_RETRIES'] || 3)
 
         ##
         # Instance methods
@@ -29,7 +29,6 @@ class Zmq < Base
         #
         def send_request
           setup_connection
-          connect_to_rpc_server
           poll_send_data
         ensure
           close_connection
@@ -77,17 +76,19 @@ def error?
         # of retries, fail the request.
         #
         def poll_send_data
-          poll_timeout = (options[:timeout] / CLIENT_RETRIES) * 1000
+          poll_timeout = (options[:timeout].to_f / CLIENT_RETRIES.to_f) * 1000
 
           CLIENT_RETRIES.times do |n|
+            connect_to_rpc_server
+            log_debug { sign_message("Sending Request (attempt #{n + 1}, #{socket})") }
             send_data
+            log_debug { sign_message("Request sending complete (attempt #{n + 1}, #{socket})") }
 
             if poller.poll(poll_timeout) == 1
               read_response
               return
             else
               socket_close
-              connect_to_rpc_server
             end
           end
 
@@ -117,10 +118,8 @@ def read_response
         # Send the request data to the remote rpc_server.
         #
         def send_data
-          log_debug { sign_message("Sending Request: #{@request_data}") }
           @stats.request_size = @request_data.size
           zmq_error_check(socket.send_string(@request_data), :socket_send_string)
-          log_debug { sign_message("write closed") }
         end
 
         # Setup a ZMQ request socket in the current zmq context.
@@ -132,7 +131,6 @@ def socket
         def socket_close
           if socket
             log_debug { sign_message("Closing Socket")  }
-            zmq_error_check(poller.deregister_readable(socket), :poller_deregister_readable)
             zmq_error_check(socket.close, :socket_close)
             log_debug { sign_message("Socket closed")  }
             @socket = nil
@@ -157,7 +155,7 @@ def zmq_context_terminate
         end
 
         def zmq_error_check(return_code, source)
-          unless ::ZMQ::Util.resultcode_ok?(return_code)
+          unless ::ZMQ::Util.resultcode_ok?(return_code || -1)
             raise <<-ERROR
             Last ZMQ API call to #{source} failed with "#{::ZMQ::Util.error_string}".
 
diff --git a/lib/protobuf/rpc/servers/zmq/broker.rb b/lib/protobuf/rpc/servers/zmq/broker.rb
index 905f2722..df7cab97 100644
--- a/lib/protobuf/rpc/servers/zmq/broker.rb
+++ b/lib/protobuf/rpc/servers/zmq/broker.rb
@@ -67,7 +67,7 @@ def move_to_backend(socket)
 
           def move_to_frontend(socket)
             message_array = []
-            zmq_error_check(socket.recv_strings(message_array), :socked_recv_strings)
+            zmq_error_check(socket.recv_strings(message_array), :socket_recv_strings)
 
             # Push UUID of socket on the available workers queue
             available_workers << message_array[0]
diff --git a/lib/protobuf/rpc/servers/zmq/util.rb b/lib/protobuf/rpc/servers/zmq/util.rb
index 2209c409..d724c730 100644
--- a/lib/protobuf/rpc/servers/zmq/util.rb
+++ b/lib/protobuf/rpc/servers/zmq/util.rb
@@ -21,7 +21,12 @@ def zmq_error_check(return_code, source)
         end
 
         def log_signature
-          @_log_signature ||= "server-#{self.class}-#{object_id}"
+          unless @_log_signature
+            name = (self.class == Class ? self.name : self.class.name)
+            @_log_signature = "[server-#{name}-#{object_id}]"
+          end
+
+          @_log_signature
         end
 
         def resolve_ip(hostname)

From c7e2daee804971d48f5289cb5a5b8dad1739ef36 Mon Sep 17 00:00:00 2001
From: BJ Neilsen <bj.neilsen@gmail.com>
Date: Wed, 20 Feb 2013 16:38:46 -0700
Subject: [PATCH 07/70] Do not send data from zmq connector when in error

---
 lib/protobuf/rpc/connectors/zmq.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index 34c9488a..c5185d77 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -76,6 +76,8 @@ def error?
         # of retries, fail the request.
         #
         def poll_send_data
+          return if error?
+
           poll_timeout = (options[:timeout].to_f / CLIENT_RETRIES.to_f) * 1000
 
           CLIENT_RETRIES.times do |n|
@@ -118,6 +120,8 @@ def read_response
         # Send the request data to the remote rpc_server.
         #
         def send_data
+          return if error?
+
           @stats.request_size = @request_data.size
           zmq_error_check(socket.send_string(@request_data), :socket_send_string)
         end

From 247368c8a08f66b7dbda18c2919a0261ba65bef6 Mon Sep 17 00:00:00 2001
From: BJ Neilsen <bj.neilsen@gmail.com>
Date: Thu, 21 Feb 2013 09:59:25 -0700
Subject: [PATCH 08/70] Specify server type in benchmark task

---
 spec/benchmark/tasks.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spec/benchmark/tasks.rb b/spec/benchmark/tasks.rb
index 2db4450a..c46b1f38 100644
--- a/spec/benchmark/tasks.rb
+++ b/spec/benchmark/tasks.rb
@@ -33,7 +33,7 @@ def em_client_em_server(number_tests, test_length, global_bench = nil)
     EM.stop if EM.reactor_running?
 
     EventMachine.fiber_run do
-      StubServer.new do |server|
+      StubServer.new(:server => Protobuf::Rpc::Evented::Server) do |server|
         benchmark_wrapper(global_bench) do |bench|
           bench.report("ES / EC") do
             (1..number_tests.to_i).each do

From 40bfb760e4c5a79ecbab9744c8f489c76291cdfa Mon Sep 17 00:00:00 2001
From: BJ Neilsen <bj.neilsen@gmail.com>
Date: Thu, 21 Feb 2013 10:24:36 -0700
Subject: [PATCH 09/70] Add zmq test (failing) to test client timeouts

---
 spec/functional/zmq_server_spec.rb    | 58 ++++++++++++++++++---------
 spec/support/test/resource.pb.rb      |  6 +++
 spec/support/test/resource.proto      |  5 +++
 spec/support/test/resource_service.rb |  7 ++++
 4 files changed, 57 insertions(+), 19 deletions(-)

diff --git a/spec/functional/zmq_server_spec.rb b/spec/functional/zmq_server_spec.rb
index 80a1195d..469a7425 100644
--- a/spec/functional/zmq_server_spec.rb
+++ b/spec/functional/zmq_server_spec.rb
@@ -25,31 +25,51 @@
     }.to_not raise_error
   end
 
-  it 'calls the on_failure callback when a message is malformed' do
-    error = nil
-    StubServer.new(:server => Protobuf::Rpc::Zmq::Server) do
-      request = ::Test::ResourceFindRequest.new(:active => true)
-      client = ::Test::ResourceService.client
-
-      client.find(request) do |c|
-        c.on_success { raise "shouldn't pass" }
-        c.on_failure {|e| error = e }
+  context 'when a message is malformed' do
+    it 'calls the on_failure callback' do
+      error = nil
+      StubServer.new(:server => Protobuf::Rpc::Zmq::Server) do
+        request = ::Test::ResourceFindRequest.new(:active => true)
+        client = ::Test::ResourceService.client
+
+        client.find(request) do |c|
+          c.on_success { raise "shouldn't pass" }
+          c.on_failure {|e| error = e }
+        end
       end
+      error.message.should match(/name.*required/)
     end
-    error.message.should =~ /name.*required/
   end
 
-  it 'calls the on_failure callback when the request type is wrong' do
-    error = nil
-    StubServer.new(:server => Protobuf::Rpc::Zmq::Server) do
-      request = ::Test::Resource.new(:name => 'Test Name')
-      client = ::Test::ResourceService.client
+  context 'when the request type is wrong' do
+    it 'calls the on_failure callback' do
+      error = nil
+      StubServer.new(:server => Protobuf::Rpc::Zmq::Server) do
+        request = ::Test::Resource.new(:name => 'Test Name')
+        client = ::Test::ResourceService.client
 
-      client.find(request) do |c|
-        c.on_success { raise "shouldn't pass" }
-        c.on_failure {|e| error = e}
+        client.find(request) do |c|
+          c.on_success { raise "shouldn't pass" }
+          c.on_failure {|e| error = e}
+        end
       end
+      error.message.should match(/expected request.*ResourceFindRequest.*Resource instead/i)
     end
-    error.message.should =~ /expected request.*ResourceFindRequest.*Resource instead/i
   end
+
+  context 'when the server takes too long to respond' do
+    it 'responds with a timeout error' do
+      error = nil
+      StubServer.new(:server => Protobuf::Rpc::Zmq::Server) do
+        client = ::Test::ResourceService.client(:timeout => 1)
+
+        client.find_with_sleep(:sleep => 2) do |c|
+          c.on_success { raise "shouldn't pass" }
+          c.on_failure { |e| error = e }
+        end
+      end
+      error.message.should match(/The server took longer than 1 seconds to respond/i)
+    end
+  end
+
 end
diff --git a/spec/support/test/resource.pb.rb b/spec/support/test/resource.pb.rb
index 356fb47d..e6abae55 100644
--- a/spec/support/test/resource.pb.rb
+++ b/spec/support/test/resource.pb.rb
@@ -14,6 +14,7 @@ class StatusType < ::Protobuf::Enum; end
   # Message Classes
   #
   class ResourceFindRequest < ::Protobuf::Message; end
+  class ResourceSleepRequest < ::Protobuf::Message; end
   class Resource < ::Protobuf::Message; end
   class Nested < ::Protobuf::Message; end
   
@@ -35,6 +36,10 @@ class ResourceFindRequest
     optional ::Protobuf::Field::BoolField, :active, 2
   end
   
+  class ResourceSleepRequest
+    optional ::Protobuf::Field::Int32Field, :sleep, 1
+  end
+  
   class Resource
     required ::Protobuf::Field::StringField, :name, 1
     optional ::Protobuf::Field::Int64Field, :date_created, 2
@@ -54,5 +59,6 @@ class Nested
   #
   class ResourceService < ::Protobuf::Rpc::Service
     rpc :find, ::Test::ResourceFindRequest, ::Test::Resource
+    rpc :find_with_sleep, ::Test::ResourceSleepRequest, ::Test::Resource
   end
 end
diff --git a/spec/support/test/resource.proto b/spec/support/test/resource.proto
index 202df04d..ba9f502d 100644
--- a/spec/support/test/resource.proto
+++ b/spec/support/test/resource.proto
@@ -12,6 +12,10 @@ message ResourceFindRequest {
   optional bool active = 2;
 }
 
+message ResourceSleepRequest {
+  optional int32 sleep = 1;
+}
+
 message Resource {
   required string name = 1;
   optional int64 date_created = 2;
@@ -28,4 +32,5 @@ message Nested {
 
 service ResourceService {
   rpc Find (ResourceFindRequest) returns (Resource);
+  rpc FindWithSleep (ResourceSleepRequest) returns (Resource);
 }
diff --git a/spec/support/test/resource_service.rb b/spec/support/test/resource_service.rb
index f1dab010..33072ba4 100644
--- a/spec/support/test/resource_service.rb
+++ b/spec/support/test/resource_service.rb
@@ -10,5 +10,12 @@ def find
       response.status = request.active ? 1 : 0
     end
 
+    # request -> Test::ResourceSleepRequest
+    # response -> Test::Resource
+    def find_with_sleep
+      sleep (request.sleep || 1)
+      response.name = 'Request should have timed out'
+    end
+
   end
 end

From f540ab691d2530a8efcb64425af9491e93911e4e Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Mon, 25 Mar 2013 17:18:14 -0600
Subject: [PATCH 10/70] Make runners and servers instance based

---
 lib/protobuf/cli.rb                           | 17 ++--
 lib/protobuf/rpc/servers/evented_runner.rb    | 19 +++--
 lib/protobuf/rpc/servers/socket/server.rb     | 27 ++++---
 lib/protobuf/rpc/servers/socket_runner.rb     | 41 ++++++----
 lib/protobuf/rpc/servers/zmq/broker.rb        |  2 +-
 lib/protobuf/rpc/servers/zmq/server.rb        | 40 +++++-----
 lib/protobuf/rpc/servers/zmq/worker.rb        |  9 ++-
 lib/protobuf/rpc/servers/zmq_runner.rb        | 42 +++++-----
 spec/functional/socket_server_spec.rb         | 11 +--
 spec/functional/zmq_server_spec.rb            | 20 ++---
 spec/lib/protobuf/cli_spec.rb                 | 78 ++++++++++---------
 .../rpc/servers/evented_server_spec.rb        |  3 +-
 .../rpc/servers/socket_server_spec.rb         | 17 ++--
 .../protobuf/rpc/servers/zmq/server_spec.rb   | 25 +++---
 spec/support/server.rb                        | 21 ++---
 15 files changed, 208 insertions(+), 164 deletions(-)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index af4e2060..791d27ff 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -171,7 +171,7 @@ def run_if_no_abort
       end
 
       def runner_options
-        { 
+        {
           :host => options.host,
           :port => options.port,
           :backlog => options.backlog,
@@ -187,7 +187,7 @@ def say_and_exit!(message, exception = nil)
 
         ::Protobuf::Logger.error { message }
         if exception
-          $stderr.puts "[#{exception.class.name}] #{exception.message}" 
+          $stderr.puts "[#{exception.class.name}] #{exception.message}"
           $stderr.puts exception.backtrace.join("\n")
 
           ::Protobuf::Logger.error { "[#{exception.class.name}] #{exception.message}" }
@@ -199,17 +199,17 @@ def say_and_exit!(message, exception = nil)
 
       def server_evented!
         @mode = :evented
-        @runner = ::Protobuf::Rpc::EventedRunner
+        @runner = ::Protobuf::Rpc::EventedRunner.new(runner_options)
       end
 
       def server_socket!
         @mode = :socket
-        @runner = ::Protobuf::Rpc::SocketRunner
+        @runner = ::Protobuf::Rpc::SocketRunner.new(runner_options)
       end
 
       def server_zmq!
         @mode = :zmq
-        @runner = ::Protobuf::Rpc::ZmqRunner
+        @runner = ::Protobuf::Rpc::ZmqRunner.new(runner_options)
       end
 
       # Start the runner and log the relevant options.
@@ -217,15 +217,14 @@ def start_server!
         @runner.register_signals
 
         debug_say 'Invoking server start'
-        @runner.run(runner_options) do
-          ::Protobuf::Logger.info { 
+
+        @runner.run do
+          ::Protobuf::Logger.info {
             "pid #{::Process.pid} -- #{@mode} RPC Server listening at #{options.host}:#{options.port}"
           }
         end
       end
-
     end
-
   end
 end
 
diff --git a/lib/protobuf/rpc/servers/evented_runner.rb b/lib/protobuf/rpc/servers/evented_runner.rb
index 0825647d..f932f92f 100644
--- a/lib/protobuf/rpc/servers/evented_runner.rb
+++ b/lib/protobuf/rpc/servers/evented_runner.rb
@@ -2,22 +2,31 @@ module Protobuf
   module Rpc
     class EventedRunner
 
-      def self.register_signals
+      def initialize(options)
+        @options = options
+      end
+
+      def register_signals
         # Noop
       end
 
-      def self.run(options)
+      def run
         # Startup and run the rpc server
         ::EventMachine.schedule do
-          ::EventMachine.start_server(options[:host], options[:port], ::Protobuf::Rpc::Evented::Server)
+          ::EventMachine.start_server(
+            @options[:host],
+            @options[:port],
+            ::Protobuf::Rpc::Evented::Server
+          )
         end
 
         # Join or start the reactor
-				yield if block_given?
+        yield if block_given?
+
         ::EM.reactor_running? ? ::EM.reactor_thread.join : ::EM.run
       end
 
-      def self.stop
+      def stop
         ::EventMachine.stop_event_loop if ::EventMachine.reactor_running?
       end
 
diff --git a/lib/protobuf/rpc/servers/socket/server.rb b/lib/protobuf/rpc/servers/socket/server.rb
index 1d67f4ee..ed9ae980 100644
--- a/lib/protobuf/rpc/servers/socket/server.rb
+++ b/lib/protobuf/rpc/servers/socket/server.rb
@@ -11,12 +11,16 @@ class Server
 
         AUTO_COLLECT_TIMEOUT = 5 # seconds
 
-        def self.cleanup?
+        def initialize(options)
+          @options = options
+        end
+
+        def cleanup?
           # every 10 connections run a cleanup routine after closing the response
           @threads.size > (@threshold - 1) && (@threads.size % @threshold) == 0
         end
 
-        def self.cleanup_threads
+        def cleanup_threads
           log_debug { sign_message("Thread cleanup - #{@threads.size} - start") }
 
           @threads = @threads.select do |t|
@@ -32,11 +36,11 @@ def self.cleanup_threads
           log_debug { sign_message("Thread cleanup - #{@threads.size} - complete") }
         end
 
-        def self.log_signature
+        def log_signature
           @_log_signature ||= "server-#{self.class.name}"
         end
 
-        def self.new_worker(socket)
+        def new_worker(socket)
           Thread.new(socket) do |sock|
             ::Protobuf::Rpc::Socket::Worker.new(sock) do |s|
               s.close
@@ -44,12 +48,12 @@ def self.new_worker(socket)
           end
         end
 
-        def self.run(options = {})
+        def run
           log_debug { sign_message("Run") }
-          host = options[:host]
-          port = options[:port]
-          backlog = options[:backlog]
-          @threshold = options[:threshold]
+          host = @options[:host]
+          port = @options[:port]
+          backlog = @options[:backlog]
+          @threshold = @options[:threshold]
 
           @threads = []
           @server = ::TCPServer.new(host, port)
@@ -96,16 +100,15 @@ def self.run(options = {})
           raise #if running?
         end
 
-        def self.running?
+        def running?
           @running
         end
 
-        def self.stop
+        def stop
           @running = false
           @server.close if @server
         end
       end
-
     end
   end
 end
diff --git a/lib/protobuf/rpc/servers/socket_runner.rb b/lib/protobuf/rpc/servers/socket_runner.rb
index d35cdcbb..9b376961 100644
--- a/lib/protobuf/rpc/servers/socket_runner.rb
+++ b/lib/protobuf/rpc/servers/socket_runner.rb
@@ -2,30 +2,37 @@ module Protobuf
   module Rpc
     class SocketRunner
 
-      def self.register_signals
-        # noop
+      def initialize(options)
+        @options = case
+                   when options.is_a?(OpenStruct) then
+                     options.marshal_dump
+                   when options.is_a?(Hash) then
+                     options
+                   when options.respond_to?(:to_hash) then
+                     options.to_hash
+                   else
+                     raise "Cannot parser Socket Server - server options"
+                   end
+
+        @server = ::Protobuf::Rpc::Socket::Server.new(@options)
       end
 
-      def self.run(server)
-        server_config = case
-                        when server.is_a?(OpenStruct) then
-                          server.marshal_dump
-                        when server.is_a?(Hash) then
-                          server
-                        when server.respond_to?(:to_hash) then
-                          server.to_hash
-                        else
-                          raise "Cannot parser Socket Server - server options"
-                        end
+      def register_signals
+        # noop
+      end
 
-				yield if block_given?
-        ::Protobuf::Rpc::Socket::Server.run(server_config)
+      def run
+        yield if block_given?
+        @server.run
       end
 
-      def self.stop
-        ::Protobuf::Rpc::Socket::Server.stop
+      def running?
+        @server.running?
       end
 
+      def stop
+        @server.stop
+      end
     end
   end
 end
diff --git a/lib/protobuf/rpc/servers/zmq/broker.rb b/lib/protobuf/rpc/servers/zmq/broker.rb
index e001ee23..f6604bec 100644
--- a/lib/protobuf/rpc/servers/zmq/broker.rb
+++ b/lib/protobuf/rpc/servers/zmq/broker.rb
@@ -25,7 +25,7 @@ def initialize(options = {})
         #
         def poll
           if frontend.nil?
-            if local_workers_have_started?            
+            if local_workers_have_started?
               # only open the front end when the workers are done booting
               log_info { "Starting frontend socket in broker, all workers ready!" }
               setup_frontend 
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 1d9ab259..30d4f6b5 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -8,18 +8,24 @@ module Zmq
       class Server
         include ::Protobuf::Rpc::Zmq::Util
 
-        ##
-        # Class Methods
-        #
-        def self.run(options = {})
+        attr_accessor :threads, :options
+
+        def initialize(options)
           @options = options
+          @threads = []
+        end
 
-          unless options[:workers_only]
+        def host
+          @options[:host]
+        end
+
+        def run
+          unless @options[:workers_only]
             log_debug { sign_message("initializing broker") }
-            @broker = ::Protobuf::Rpc::Zmq::Broker.new(options)
+            @broker = ::Protobuf::Rpc::Zmq::Broker.new(@options)
           end
 
-          local_worker_threads = options[:threads]
+          local_worker_threads = @options[:threads]
           log_debug { sign_message("starting server workers") }
 
           local_worker_threads.times do
@@ -29,7 +35,7 @@ def self.run(options = {})
           @running = true
           log_debug { sign_message("server started") }
           while self.running? do
-            if options[:workers_only]
+            if @options[:workers_only]
               sleep 5
               Thread.pass
             else
@@ -40,25 +46,25 @@ def self.run(options = {})
           @broker.teardown if @broker
         end
 
-        def self.running?
+        def running?
           !!@running
         end
 
-        def self.start_worker
-          @threads << Thread.new(@options) { |options|
+        def start_worker
+          @threads << Thread.new(self) { |server|
             begin
-              ::Protobuf::Rpc::Zmq::Worker.new(options).run
+              ::Protobuf::Rpc::Zmq::Worker.new(server).run
             rescue => e
               message =  "Worker Failed, spawning new worker: #{e.inspect}\n #{e.backtrace.join($/)}"
               $stderr.puts message
               log_error { message }
 
-              retry if ::Protobuf::Rpc::Zmq::Server.running?
+              retry if self.running?
             end
           }
         end
 
-        def self.stop
+        def stop
           @running = false
 
           @threads.each do |t|
@@ -66,11 +72,9 @@ def self.stop
           end
         end
 
-        def self.threads
-          @threads
+        def worker_port
+          @options[:worker_port]
         end
-
-        @threads ||= []
       end
     end
   end
diff --git a/lib/protobuf/rpc/servers/zmq/worker.rb b/lib/protobuf/rpc/servers/zmq/worker.rb
index c0c69053..f2b50eb8 100644
--- a/lib/protobuf/rpc/servers/zmq/worker.rb
+++ b/lib/protobuf/rpc/servers/zmq/worker.rb
@@ -11,9 +11,10 @@ class Worker
         ##
         # Constructor
         #
-        def initialize(options = {})
-          host = options[:host]
-          port = options[:worker_port]
+        def initialize(server)
+          @server = server
+          host = server.host
+          port = server.worker_port
 
           @zmq_context = ::ZMQ::Context.new
           @socket = @zmq_context.socket(::ZMQ::REQ)
@@ -39,7 +40,7 @@ def handle_request(socket)
         end
 
         def run
-          while ::Protobuf::Rpc::Zmq::Server.running? do
+          while @server.running? do
             # poll for 1_000 milliseconds then continue looping
             # This lets us see whether we need to die
             @poller.poll(1_000)
diff --git a/lib/protobuf/rpc/servers/zmq_runner.rb b/lib/protobuf/rpc/servers/zmq_runner.rb
index b2f4db20..b3d63cb5 100644
--- a/lib/protobuf/rpc/servers/zmq_runner.rb
+++ b/lib/protobuf/rpc/servers/zmq_runner.rb
@@ -3,33 +3,39 @@ module Rpc
     class ZmqRunner
       include ::Protobuf::Logger::LogMethods
 
-      def self.register_signals
-        trap(:TTIN) do 
+      def initialize(options)
+        @options = case
+                   when options.is_a?(OpenStruct) then
+                     options.marshal_dump
+                   when options.respond_to?(:to_hash) then
+                     options.to_hash
+                   else
+                     raise "Cannot parser Zmq Server - server options"
+                   end
+
+        @server = ::Protobuf::Rpc::Zmq::Server.new(@options)
+      end
+
+      def register_signals
+        trap(:TTIN) do
           log_info { "TTIN received: Starting new worker" }
-          ::Protobuf::Rpc::Zmq::Server.start_worker
+          @server.start_worker
           log_info { "Worker count : #{::Protobuf::Rpc::Zmq::Server.threads.size}" }
         end
       end
 
-      def self.run(server)
-        server_config = case
-                        when server.is_a?(OpenStruct) then
-                          server.marshal_dump
-                        when server.respond_to?(:to_hash) then
-                          server.to_hash
-                        else
-                          raise "Cannot parser Zmq Server - server options"
-                        end
-
-				yield if block_given?
-
-        ::Protobuf::Rpc::Zmq::Server.run(server_config)
+      def run
+        yield if block_given?
+        @server.run
       end
 
-      def self.stop
-        ::Protobuf::Rpc::Zmq::Server.stop
+      def running?
+        @server.running?
       end
 
+      def stop
+        @server.stop
+      end
     end
   end
 end
diff --git a/spec/functional/socket_server_spec.rb b/spec/functional/socket_server_spec.rb
index 02ce00aa..2af158d1 100644
--- a/spec/functional/socket_server_spec.rb
+++ b/spec/functional/socket_server_spec.rb
@@ -4,14 +4,14 @@
 describe 'Functional Socket Client' do
   before(:all) do
     load "protobuf/socket.rb"
-    Thread.abort_on_exception = true
-    server = OpenStruct.new(:host => "127.0.0.1", :port => 9399, :backlog => 100, :threshold => 100)
-    @server_thread = Thread.new(server) { |s| Protobuf::Rpc::SocketRunner.run(s) }
-    Thread.pass until Protobuf::Rpc::Socket::Server.running?
+    @options = OpenStruct.new(:host => "127.0.0.1", :port => 9399, :backlog => 100, :threshold => 100)
+    @runner = ::Protobuf::Rpc::SocketRunner.new(@options)
+    @server_thread = Thread.new(@runner) { |runner| runner.run }
+    Thread.pass until @runner.running?
   end
 
   after(:all) do
-    Protobuf::Rpc::SocketRunner.stop
+    @runner.stop
     @server_thread.join
   end
 
@@ -56,3 +56,4 @@
     error.message.should =~ /expected request.*ResourceFindRequest.*Resource instead/i
   end
 end
+
diff --git a/spec/functional/zmq_server_spec.rb b/spec/functional/zmq_server_spec.rb
index ef02c16c..302e3eda 100644
--- a/spec/functional/zmq_server_spec.rb
+++ b/spec/functional/zmq_server_spec.rb
@@ -5,19 +5,21 @@
   before(:all) do
     load "protobuf/zmq.rb"
     Thread.abort_on_exception = true
-    server = OpenStruct.new(:host => "127.0.0.1", 
-                            :port => 9399, 
-                            :worker_port => 9400, 
-                            :backlog => 100, 
-                            :threshold => 100, 
-                            :threads => 5)
+    options = OpenStruct.new(:host => "127.0.0.1",
+                             :port => 9399,
+                             :worker_port => 9400,
+                             :backlog => 100,
+                             :threshold => 100,
+                             :threads => 5)
 
-    @server_thread = Thread.new(server) { |s| Protobuf::Rpc::ZmqRunner.run(s) }
-    Thread.pass until Protobuf::Rpc::Zmq::Server.running?
+    @runner = ::Protobuf::Rpc::ZmqRunner.new(options)
+    @server_thread = Thread.new(@runner) { |runner| runner.run }
+
+    Thread.pass until @runner.running?
   end
 
   after(:all) do
-    ::Protobuf::Rpc::Zmq::Server.stop
+    @runner.stop
     @server_thread.try(:join)
   end
 
diff --git a/spec/lib/protobuf/cli_spec.rb b/spec/lib/protobuf/cli_spec.rb
index 2bfef524..0de5e5ed 100644
--- a/spec/lib/protobuf/cli_spec.rb
+++ b/spec/lib/protobuf/cli_spec.rb
@@ -7,10 +7,14 @@
     File.expand_path('../../../support/test_app_file.rb', __FILE__)
   end
 
-  before do
-    ::Protobuf::Rpc::SocketRunner.stub(:run)
-    ::Protobuf::Rpc::ZmqRunner.stub(:run)
-    ::Protobuf::Rpc::EventedRunner.stub(:run)
+  let(:sock_runner) { double "SocketRunner", run: nil, register_signals: nil }
+  let(:zmq_runner) { double "ZmqRunner", run: nil, register_signals: nil }
+  let(:evented_runner) { double "EventedRunner", run: nil, register_signals: nil }
+
+  before(:each) do
+    ::Protobuf::Rpc::SocketRunner.stub(:new) { sock_runner }
+    ::Protobuf::Rpc::ZmqRunner.stub(:new) { zmq_runner }
+    ::Protobuf::Rpc::EventedRunner.stub(:new) { evented_runner }
   end
 
   describe '#start' do
@@ -22,9 +26,9 @@
       let(:test_args) { [ '--host=123.123.123.123' ] }
 
       it 'sends the host option to the runner' do
-        ::Protobuf::Rpc::SocketRunner.should_receive(:run) do |options|
+        ::Protobuf::Rpc::SocketRunner.should_receive(:new) do |options|
           options[:host].should eq '123.123.123.123'
-        end
+        end.and_return(sock_runner)
         described_class.start(args)
       end
     end
@@ -33,9 +37,9 @@
       let(:test_args) { [ '--port=12345' ] }
 
       it 'sends the port option to the runner' do
-        ::Protobuf::Rpc::SocketRunner.should_receive(:run) do |options|
+        ::Protobuf::Rpc::SocketRunner.should_receive(:new) do |options|
           options[:port].should eq 12345
-        end
+        end.and_return(sock_runner)
         described_class.start(args)
       end
     end
@@ -44,9 +48,9 @@
       let(:test_args) { [ '--threads=500' ] }
 
       it 'sends the threads option to the runner' do
-        ::Protobuf::Rpc::SocketRunner.should_receive(:run) do |options|
+        ::Protobuf::Rpc::SocketRunner.should_receive(:new) do |options|
           options[:threads].should eq 500
-        end
+        end.and_return(sock_runner)
         described_class.start(args)
       end
     end
@@ -55,9 +59,9 @@
       let(:test_args) { [ '--backlog=500' ] }
 
       it 'sends the backlog option to the runner' do
-        ::Protobuf::Rpc::SocketRunner.should_receive(:run) do |options|
+        ::Protobuf::Rpc::SocketRunner.should_receive(:new) do |options|
           options[:backlog].should eq 500
-        end
+        end.and_return(sock_runner)
         described_class.start(args)
       end
     end
@@ -66,9 +70,9 @@
       let(:test_args) { [ '--threshold=500' ] }
 
       it 'sends the backlog option to the runner' do
-        ::Protobuf::Rpc::SocketRunner.should_receive(:run) do |options|
+        ::Protobuf::Rpc::SocketRunner.should_receive(:new) do |options|
           options[:threshold].should eq 500
-        end
+        end.and_return(sock_runner)
         described_class.start(args)
       end
     end
@@ -156,18 +160,18 @@
         let(:runner) { ::Protobuf::Rpc::SocketRunner }
 
         before do
-          ::Protobuf::Rpc::EventedRunner.should_not_receive(:run)
-          ::Protobuf::Rpc::ZmqRunner.should_not_receive(:run)
+          ::Protobuf::Rpc::EventedRunner.should_not_receive(:new)
+          ::Protobuf::Rpc::ZmqRunner.should_not_receive(:new)
         end
 
         it 'is activated by the --socket switch' do
-          runner.should_receive(:run)
+          runner.should_receive(:new)
           described_class.start(args)
         end
 
         it 'is activated by PB_SERVER_TYPE=Socket ENV variable' do 
           ENV['PB_SERVER_TYPE'] = "Socket"
-          runner.should_receive(:run)
+          runner.should_receive(:new).and_return(sock_runner)
           described_class.start(args)
           ENV.delete('PB_SERVER_TYPE')
         end
@@ -183,18 +187,18 @@
         let(:runner) { ::Protobuf::Rpc::EventedRunner }
 
         before do
-          ::Protobuf::Rpc::SocketRunner.should_not_receive(:run)
-          ::Protobuf::Rpc::ZmqRunner.should_not_receive(:run)
+          ::Protobuf::Rpc::SocketRunner.should_not_receive(:new)
+          ::Protobuf::Rpc::ZmqRunner.should_not_receive(:new)
         end
 
         it 'is activated by the --evented switch' do
-          runner.should_receive(:run)
+          runner.should_receive(:new).and_return(evented_runner)
           described_class.start(args)
         end
 
         it 'is activated by PB_SERVER_TYPE=Evented ENV variable' do 
           ENV['PB_SERVER_TYPE'] = "Evented"
-          runner.should_receive(:run)
+          runner.should_receive(:new).and_return(evented_runner)
           described_class.start(args)
           ENV.delete('PB_SERVER_TYPE')
         end
@@ -204,29 +208,29 @@
           ::Protobuf.connector_type.should == :evented
         end
       end
-      
+
       context 'zmq workers only' do
         let(:test_args) { [ '--workers_only', '--zmq' ] }
         let(:runner) { ::Protobuf::Rpc::ZmqRunner }
 
         before do
-          ::Protobuf::Rpc::SocketRunner.should_not_receive(:run)
-          ::Protobuf::Rpc::EventedRunner.should_not_receive(:run)
+          ::Protobuf::Rpc::SocketRunner.should_not_receive(:new)
+          ::Protobuf::Rpc::EventedRunner.should_not_receive(:new)
         end
 
         it 'is activated by the --workers_only switch' do
-          runner.should_receive(:run) do |options|
+          runner.should_receive(:new) do |options|
             options[:workers_only].should be_true
-          end
+          end.and_return(zmq_runner)
 
           described_class.start(args)
         end
 
         it 'is activated by PB_WORKERS_ONLY=1 ENV variable' do 
           ENV['PB_WORKERS_ONLY'] = "1"
-          runner.should_receive(:run) do |options|
+          runner.should_receive(:new) do |options|
             options[:workers_only].should be_true
-          end
+          end.and_return(zmq_runner)
 
           described_class.start(args)
           ENV.delete('PB_WORKERS_ONLY')
@@ -238,14 +242,14 @@
         let(:runner) { ::Protobuf::Rpc::ZmqRunner }
 
         before do
-          ::Protobuf::Rpc::SocketRunner.should_not_receive(:run)
-          ::Protobuf::Rpc::EventedRunner.should_not_receive(:run)
+          ::Protobuf::Rpc::SocketRunner.should_not_receive(:new)
+          ::Protobuf::Rpc::EventedRunner.should_not_receive(:new)
         end
 
         it 'is activated by the --worker_port switch' do
-          runner.should_receive(:run) do |options|
+          runner.should_receive(:new) do |options|
             options[:worker_port].should eq(1234)
-          end
+          end.and_return(zmq_runner)
 
           described_class.start(args)
         end
@@ -256,18 +260,18 @@
         let(:runner) { ::Protobuf::Rpc::ZmqRunner }
 
         before do
-          ::Protobuf::Rpc::SocketRunner.should_not_receive(:run)
-          ::Protobuf::Rpc::EventedRunner.should_not_receive(:run)
+          ::Protobuf::Rpc::SocketRunner.should_not_receive(:new)
+          ::Protobuf::Rpc::EventedRunner.should_not_receive(:new)
         end
 
         it 'is activated by the --zmq switch' do
-          runner.should_receive(:run)
+          runner.should_receive(:new)
           described_class.start(args)
         end
 
         it 'is activated by PB_SERVER_TYPE=Zmq ENV variable' do 
           ENV['PB_SERVER_TYPE'] = "Zmq"
-          runner.should_receive(:run)
+          runner.should_receive(:new)
           described_class.start(args)
           ENV.delete('PB_SERVER_TYPE')
         end
diff --git a/spec/lib/protobuf/rpc/servers/evented_server_spec.rb b/spec/lib/protobuf/rpc/servers/evented_server_spec.rb
index 9ed8b546..558044ce 100644
--- a/spec/lib/protobuf/rpc/servers/evented_server_spec.rb
+++ b/spec/lib/protobuf/rpc/servers/evented_server_spec.rb
@@ -12,7 +12,8 @@
 
   it "Runner provides a stop method" do
     runner_class = described_class.to_s.gsub(/Evented::Server/, "EventedRunner").constantize
-    runner_class.respond_to?(:stop).should be_true
+    runner = runner_class.new({})
+    runner.respond_to?(:stop).should be_true
   end
 
 end
diff --git a/spec/lib/protobuf/rpc/servers/socket_server_spec.rb b/spec/lib/protobuf/rpc/servers/socket_server_spec.rb
index 7f7e1d5e..a0be83b7 100644
--- a/spec/lib/protobuf/rpc/servers/socket_server_spec.rb
+++ b/spec/lib/protobuf/rpc/servers/socket_server_spec.rb
@@ -12,23 +12,24 @@
   before(:all) do
     load 'protobuf/socket.rb'
     Thread.abort_on_exception = true
-    server = OpenStruct.new(:server => "127.0.0.1", :port => 9399, :backlog => 100, :threshold => 100)
-    @server_thread = Thread.new(server) { |s| Protobuf::Rpc::SocketRunner.run(s) }
-    Thread.pass until Protobuf::Rpc::Socket::Server.running?
+    @options = OpenStruct.new(:host => "127.0.0.1", :port => 9399, :backlog => 100, :threshold => 100)
+    @runner = ::Protobuf::Rpc::SocketRunner.new(@options)
+    @server = @runner.instance_variable_get(:@server)
+    @server_thread = Thread.new(@runner) { |runner| runner.run }
+    Thread.pass until @server.running?
   end
 
   after(:all) do
-    Protobuf::Rpc::SocketRunner.stop
+    @server.stop
     @server_thread.join
   end
 
   it "Runner provides a stop method" do
-    runner_class = described_class.to_s.gsub(/Evented::Server/, "EventedRunner").constantize
-    runner_class.respond_to?(:stop).should be_true
+    @runner.respond_to?(:stop).should be_true
   end
 
   it "provides a stop method" do
-    described_class.respond_to?(:stop).should be_true
+    @runner.respond_to?(:stop).should be_true
   end
 
   it "provides a Runner class" do
@@ -37,7 +38,7 @@
   end
 
   it "signals the Server is running" do
-    described_class.running?.should be_true
+    @server.running?.should be_true
   end
 
 end
diff --git a/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb b/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb
index 33937a30..bf93393a 100644
--- a/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb
+++ b/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb
@@ -2,19 +2,22 @@
 require 'protobuf/rpc/servers/zmq/server'
 
 describe Protobuf::Rpc::Zmq::Server do
-  before(:each) do 
+  subject { described_class.new({}) }
+
+  before(:each) do
     load 'protobuf/zmq.rb'
   end
 
   describe '.running?' do
+
     it 'returns true if running' do
-      described_class.instance_variable_set(:@running, true)
-      described_class.running?.should be_true
+      subject.instance_variable_set(:@running, true)
+      subject.running?.should be_true
     end
 
     it 'returns false if not running' do
-      described_class.instance_variable_set(:@running, false)
-      described_class.running?.should be_false
+      subject.instance_variable_set(:@running, false)
+      subject.running?.should be_false
     end
   end
 
@@ -22,20 +25,20 @@
     # keep threads instance variable from retaining any thread mocks we've
     # created (breaks tests down the line, otherwise)
     after(:each) do
-      described_class.instance_variable_set(:@threads, [])
+      subject.instance_variable_set(:@threads, [])
     end
 
     it 'lets all threads stop' do
       thread_mock = double(Thread)
       thread_mock.should_receive(:join).and_return(thread_mock)
-      described_class.instance_variable_set(:@threads, [thread_mock])
-      described_class.stop
+      subject.instance_variable_set(:@threads, [thread_mock])
+      subject.stop
     end
 
     it 'sets running to false' do
-      described_class.instance_variable_set(:@threads, [])
-      described_class.stop
-      described_class.instance_variable_get(:@running).should be_false
+      subject.instance_variable_set(:@threads, [])
+      subject.stop
+      subject.instance_variable_get(:@running).should be_false
     end
   end
 end
diff --git a/spec/support/server.rb b/spec/support/server.rb
index d961a0c9..6095a311 100644
--- a/spec/support/server.rb
+++ b/spec/support/server.rb
@@ -71,14 +71,17 @@ def start_em_server
   end
 
   def start_socket_server
-    @sock_server = Thread.new(@options) { |opt| Protobuf::Rpc::SocketRunner.run(opt) }
-    @sock_server.abort_on_exception = true # Set for testing purposes
-    Thread.pass until Protobuf::Rpc::Socket::Server.running?
+    @sock_runner = ::Protobuf::Rpc::SocketRunner.new(opt)
+    @sock_thread = Thread.new(@sock_runner) { |runner| runner.run }
+    @sock_thread.abort_on_exception = true # Set for testing purposes
+    Thread.pass until @sock_runner.running?
   end
 
   def start_zmq_server
-    @zmq_server = Thread.new(@options) { |opt| Protobuf::Rpc::ZmqRunner.run(opt) }
-    Thread.pass until Protobuf::Rpc::Zmq::Server.running?
+    @zmq_runnger = ::Protobuf::Rpc::ZmqRunner.new(opt)
+    @zmq_thread = Thread.new(@zmq_runner) { |runner| runner.run }
+    @zmq_thread.abort_on_exception = true # Set for testing purposes
+    Thread.pass until @zmq_runner.running?
   end
 
   def stop
@@ -86,11 +89,11 @@ def stop
     when @options.server == Protobuf::Rpc::Evented::Server then
       EventMachine.stop_server(@server_handle) if @server_handle
     when @options.server == Protobuf::Rpc::Zmq::Server then
-      Protobuf::Rpc::ZmqRunner.stop
-      @zmq_server.join if @zmq_server
+      @zmq_runner.stop
+      @zmq_thread.join if @zmq_thread
     else
-      Protobuf::Rpc::SocketRunner.stop
-      @sock_server.join if @sock_server
+      @sock_runner.stop
+      @sock_thread.join if @sock_thread
     end
 
     @running = false

From 4d6db0edaf5ff178b4839bc2eb2abdd49ad69f8a Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 27 Mar 2013 09:23:57 -0600
Subject: [PATCH 11/70] Prevent pre-mature worker shutdown

By setting @running to true after the workers are started, it is
possible that the worker could check the servers running status before
it has been set, causing the worker to shutdown.
---
 lib/protobuf/rpc/servers/zmq/server.rb | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 30d4f6b5..095ad08d 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -20,6 +20,8 @@ def host
         end
 
         def run
+          @running = true
+
           unless @options[:workers_only]
             log_debug { sign_message("initializing broker") }
             @broker = ::Protobuf::Rpc::Zmq::Broker.new(@options)
@@ -28,12 +30,13 @@ def run
           local_worker_threads = @options[:threads]
           log_debug { sign_message("starting server workers") }
 
+
           local_worker_threads.times do
             self.start_worker
           end
 
-          @running = true
           log_debug { sign_message("server started") }
+
           while self.running? do
             if @options[:workers_only]
               sleep 5

From ad7a3e1e38bb10993c6db0e8b361bc7d295e3395 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Thu, 23 May 2013 13:45:29 -0600
Subject: [PATCH 12/70] Cleanup

---
 lib/protobuf/cli.rb                        |  2 --
 lib/protobuf/rpc/servers/evented_runner.rb |  4 ---
 lib/protobuf/rpc/servers/socket_runner.rb  |  4 ---
 lib/protobuf/rpc/servers/zmq/broker.rb     | 35 +++++++---------------
 lib/protobuf/rpc/servers/zmq/worker.rb     | 13 ++++----
 lib/protobuf/rpc/servers/zmq_runner.rb     | 19 +++++++-----
 6 files changed, 28 insertions(+), 49 deletions(-)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index 791d27ff..59582d32 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -214,8 +214,6 @@ def server_zmq!
 
       # Start the runner and log the relevant options.
       def start_server!
-        @runner.register_signals
-
         debug_say 'Invoking server start'
 
         @runner.run do
diff --git a/lib/protobuf/rpc/servers/evented_runner.rb b/lib/protobuf/rpc/servers/evented_runner.rb
index f932f92f..38acade0 100644
--- a/lib/protobuf/rpc/servers/evented_runner.rb
+++ b/lib/protobuf/rpc/servers/evented_runner.rb
@@ -6,10 +6,6 @@ def initialize(options)
         @options = options
       end
 
-      def register_signals
-        # Noop
-      end
-
       def run
         # Startup and run the rpc server
         ::EventMachine.schedule do
diff --git a/lib/protobuf/rpc/servers/socket_runner.rb b/lib/protobuf/rpc/servers/socket_runner.rb
index 9b376961..bd70aad2 100644
--- a/lib/protobuf/rpc/servers/socket_runner.rb
+++ b/lib/protobuf/rpc/servers/socket_runner.rb
@@ -17,10 +17,6 @@ def initialize(options)
         @server = ::Protobuf::Rpc::Socket::Server.new(@options)
       end
 
-      def register_signals
-        # noop
-      end
-
       def run
         yield if block_given?
         @server.run
diff --git a/lib/protobuf/rpc/servers/zmq/broker.rb b/lib/protobuf/rpc/servers/zmq/broker.rb
index 3b005861..b4bf1e16 100644
--- a/lib/protobuf/rpc/servers/zmq/broker.rb
+++ b/lib/protobuf/rpc/servers/zmq/broker.rb
@@ -85,38 +85,23 @@ def local_workers_have_started?
           end
 
           def move_to_backend(socket)
-            message_array = []
-            zmq_error_check(socket.recv_strings(message_array))
-
-            backend_message_set = [
-              available_workers.shift, # Worker UUID for router
-              "",
-              message_array[0], # Client UUID for return value
-              "",
-              message_array[2] # Client Message payload (request)
-            ]
-
-            zmq_error_check(backend.send_strings(backend_message_set))
+            # frames = [CLIENT_ID, "", REQUEST_DATA]
+
+            zmq_error_check(socket.recv_strings(frames = []))
+            frames = [available_workers.shift, ""] + frames
+            zmq_error_check(backend.send_strings(frames))
           end
 
           def move_to_frontend(socket)
-            message_array = []
-            zmq_error_check(socket.recv_strings(message_array))
+            # frames = [WORKER_ID, "", READY_MESSAGE | (CLIENT_ID, "", RESPONSE_DATA)]
 
-            # Push UUID of socket on the available workers queue
-            available_workers << message_array[0]
+            zmq_error_check(socket.recv_strings(frames = []))
+            available_workers << frames.shift(2)[0]
 
-            # messages should be [ "uuid of socket", "", "READY_MESSAGE || uuid of client socket"]
-            if message_array[2] == ::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE
+            if frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
               log_info { "Worker #{available_workers.size} of #{expected_worker_count} ready!" }
             else
-              frontend_message_set = [
-                message_array[2], # client UUID
-                "",
-                message_array[4] # Reply payload
-              ]
-
-              zmq_error_check(frontend.send_strings(frontend_message_set))
+              zmq_error_check(frontend.send_strings(frames))
             end
           end
 
diff --git a/lib/protobuf/rpc/servers/zmq/worker.rb b/lib/protobuf/rpc/servers/zmq/worker.rb
index f2b50eb8..8ad18b18 100644
--- a/lib/protobuf/rpc/servers/zmq/worker.rb
+++ b/lib/protobuf/rpc/servers/zmq/worker.rb
@@ -31,12 +31,14 @@ def initialize(server)
         # Instance Methods
         #
         def handle_request(socket)
-          message_array = []
-          zmq_error_check(socket.recv_strings(message_array))
+          zmq_error_check(socket.recv_strings(frames = []))
 
-          @request_data = message_array[2]
-          @client_address = message_array[0]
-          log_debug { sign_message("handling request") } unless @request_data.nil?
+          @client_address, empty, @request_data = *frames
+
+          unless @request_data.nil?
+            log_debug { sign_message("handling request") }
+            handle_client
+          end
         end
 
         def run
@@ -47,7 +49,6 @@ def run
             @poller.readables.each do |socket|
               initialize_request!
               handle_request(socket)
-              handle_client unless @request_data.nil?
             end
           end
         ensure
diff --git a/lib/protobuf/rpc/servers/zmq_runner.rb b/lib/protobuf/rpc/servers/zmq_runner.rb
index b3d63cb5..f1219d9e 100644
--- a/lib/protobuf/rpc/servers/zmq_runner.rb
+++ b/lib/protobuf/rpc/servers/zmq_runner.rb
@@ -16,15 +16,8 @@ def initialize(options)
         @server = ::Protobuf::Rpc::Zmq::Server.new(@options)
       end
 
-      def register_signals
-        trap(:TTIN) do
-          log_info { "TTIN received: Starting new worker" }
-          @server.start_worker
-          log_info { "Worker count : #{::Protobuf::Rpc::Zmq::Server.threads.size}" }
-        end
-      end
-
       def run
+        register_signals
         yield if block_given?
         @server.run
       end
@@ -36,6 +29,16 @@ def running?
       def stop
         @server.stop
       end
+
+      private
+
+      def register_signals
+        trap(:TTIN) do
+          log_info { "TTIN received: Starting new worker" }
+          @server.start_worker
+          log_info { "Worker count : #{::Protobuf::Rpc::Zmq::Server.threads.size}" }
+        end
+      end
     end
   end
 end

From 0097af1d1dfe5da6640a8690f23fb96932652260 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 24 May 2013 13:57:25 -0600
Subject: [PATCH 13/70] Add dynamic discovery option for ZMQ server

---
 lib/protobuf/cli.rb                           |  34 +--
 lib/protobuf/rpc/servers/zmq/broker.rb        | 114 --------
 lib/protobuf/rpc/servers/zmq/server.rb        | 245 +++++++++++++++---
 lib/protobuf/rpc/servers/zmq/util.rb          |   3 +-
 lib/protobuf/rpc/servers/zmq/worker.rb        |  43 +--
 proto/dynamic_discovery.proto                 |  40 +++
 .../protobuf/rpc/servers/zmq/broker_spec.rb   |  31 ---
 .../protobuf/rpc/servers/zmq/server_spec.rb   |  38 ++-
 8 files changed, 317 insertions(+), 231 deletions(-)
 delete mode 100644 lib/protobuf/rpc/servers/zmq/broker.rb
 create mode 100644 proto/dynamic_discovery.proto
 delete mode 100644 spec/lib/protobuf/rpc/servers/zmq/broker_spec.rb

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index 59582d32..809b0ffe 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -29,6 +29,7 @@ class CLI < ::Thor
     option :evented,                    :type => :boolean, :aliases => %w(-m), :desc => 'Evented Mode for server and client connections (uses EventMachine).'
     option :zmq,                        :type => :boolean, :aliases => %w(-z), :desc => 'ZeroMQ Socket Mode for server and client connections.'
 
+    option :dynamic_discovery,          :type => :boolean, :default => false, :aliases => %w(-y), :desc => 'Enable dynamic discovery (Currently only available with ZeroMQ).'
     option :debug,                      :type => :boolean, :default => false, :aliases => %w(-d), :desc => 'Debug Mode. Override log level to DEBUG.'
     option :gc_pause_request,           :type => :boolean, :default => false, :desc => 'Enable/Disable GC pause during request.'
     option :print_deprecation_warnings, :type => :boolean, :default => nil, :desc => 'Cause use of deprecated fields to be printed or ignored.'
@@ -101,25 +102,26 @@ def configure_process_name(app_file)
       # Configure the mode of the server and the runner class.
       def configure_server_mode
         debug_say 'Configuring runner mode'
-        if options.zmq? && ! options.evented? && ! options.socket?
+
+        if multi_mode?
+          say 'WARNING: You have provided multiple mode options. Defaulting to socket mode.', :yellow
+          server_socket!
+        elsif options.zmq?
           server_zmq!
-        elsif options.evented? && ! options.zmq? && ! options.socket?
+        elsif options.evented?
           server_evented!
-        elsif (env_server_type = ENV["PB_SERVER_TYPE"])
-          case
-          when env_server_type =~ /zmq/i then
-            server_zmq!
-          when env_server_type =~ /socket/i then
+        else
+          case server_type = ENV["PB_SERVER_TYPE"]
+          when nil, /socket/i
             server_socket!
-          when env_server_type =~ /evented/i then
+          when /zmq/i
+            server_zmq!
+          when /evented/i
             server_evented!
           else
-            say "WARNING: You have provided incorrect option 'PB_SERVER_TYPE=#{env_server_type}'. Defaulting to socket mode.", :yellow
+            say "WARNING: You have provided incorrect option 'PB_SERVER_TYPE=#{server_type}'. Defaulting to socket mode.", :yellow
             server_socket!
           end
-        else
-          say 'WARNING: You have provided multiple mode options. Defaulting to socket mode.', :yellow if multi_mode?
-          server_socket!
         end
       end
 
@@ -145,9 +147,11 @@ def debug_say(message, color = :yellow)
 
       # Internal helper to determine if the modes are multi-set which is not valid.
       def multi_mode?
-        (options.zmq? && (options.evented? || options.socket?)) \
-          && (options.evented? && (options.evented? || options.socket?)) \
-          && (options.zmq? && (options.evented? || options.socket?)) \
+        [
+          options.zmq?,
+          options.evented?,
+          options.socket?,
+        ].count(true) > 1
       end
 
       # Require the application file given, exiting if the file doesn't exist.
diff --git a/lib/protobuf/rpc/servers/zmq/broker.rb b/lib/protobuf/rpc/servers/zmq/broker.rb
deleted file mode 100644
index b4bf1e16..00000000
--- a/lib/protobuf/rpc/servers/zmq/broker.rb
+++ /dev/null
@@ -1,114 +0,0 @@
-require 'resolv'
-require 'protobuf/rpc/servers/zmq/util'
-
-module Protobuf
-  module Rpc
-    module Zmq
-      class Broker
-        include ::Protobuf::Rpc::Zmq::Util
-        attr_reader :frontend, :backend, :poller, :context, :available_workers, :options, :expected_worker_count
-
-        ##
-        # Constructor
-        #
-        def initialize(options = {})
-          @available_workers = []
-          @options = options.dup
-          @expected_worker_count = @options[:threads]
-          @context = ::ZMQ::Context.new
-          @poller = ::ZMQ::Poller.new
-          setup_backend
-        end
-
-        ##
-        # Instance Methods
-        #
-        def poll
-          if frontend.nil?
-            if local_workers_have_started?
-              # only open the front end when the workers are done booting
-              log_info { "Starting frontend socket in broker, all workers ready!" }
-              setup_frontend
-            end
-          else
-            # Start checking the poller after startup
-            if available_workers.size > 0
-              poller.register(frontend, ::ZMQ::POLLIN) if poller.size < 2
-            else
-              poller.delete(frontend)
-            end
-          end
-
-          poller.poll(1000)
-          poller.readables.each do |socket|
-            case socket
-            when backend then
-              move_to_frontend(socket)
-            when frontend then
-              move_to_backend(socket)
-            end
-          end
-        end
-
-        def setup_backend
-          host = options[:host]
-          port = options[:worker_port]
-
-          zmq_backend = context.socket(::ZMQ::ROUTER)
-          zmq_error_check(zmq_backend.bind(bind_address(host, port)))
-
-          @backend = zmq_backend
-          @poller.register(@backend, ::ZMQ::POLLIN)
-        end
-
-        def setup_frontend
-          host = options[:host]
-          port = options[:port]
-
-          zmq_frontend = context.socket(::ZMQ::ROUTER)
-          zmq_error_check(zmq_frontend.bind(bind_address(host, port)))
-
-          @frontend = zmq_frontend
-          @poller.register(@frontend, ::ZMQ::POLLIN)
-        end
-
-        def teardown
-          frontend.try(:close)
-          backend.try(:close)
-          context.try(:terminate)
-        end
-
-        private
-
-          def local_workers_have_started?
-            @local_workers_have_started ||= available_workers.size >= expected_worker_count
-          end
-
-          def move_to_backend(socket)
-            # frames = [CLIENT_ID, "", REQUEST_DATA]
-
-            zmq_error_check(socket.recv_strings(frames = []))
-            frames = [available_workers.shift, ""] + frames
-            zmq_error_check(backend.send_strings(frames))
-          end
-
-          def move_to_frontend(socket)
-            # frames = [WORKER_ID, "", READY_MESSAGE | (CLIENT_ID, "", RESPONSE_DATA)]
-
-            zmq_error_check(socket.recv_strings(frames = []))
-            available_workers << frames.shift(2)[0]
-
-            if frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
-              log_info { "Worker #{available_workers.size} of #{expected_worker_count} ready!" }
-            else
-              zmq_error_check(frontend.send_strings(frames))
-            end
-          end
-
-          def bind_address(host, port)
-            "tcp://#{resolve_ip(host)}:#{port}"
-          end
-      end
-    end
-  end
-end
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 5fbf0e63..c410e90c 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -1,4 +1,3 @@
-require 'protobuf/rpc/servers/zmq/broker'
 require 'protobuf/rpc/servers/zmq/worker'
 require 'protobuf/rpc/servers/zmq/util'
 
@@ -8,74 +7,248 @@ module Zmq
       class Server
         include ::Protobuf::Rpc::Zmq::Util
 
-        attr_accessor :threads, :options
+        attr_accessor :options, :workers, :zmq_context
 
         def initialize(options)
-          @options = options
-          @threads = []
+          @options = default_options.merge(options)
+          @workers = []
+
+          init_zmq_context
+          init_backend_socket
+          init_frontend_socket unless brokerless?
+          init_shutdown_socket
+          init_beacon_socket if broadcast_beacons?
         end
 
-        def host
-          @options[:host]
+        def alive_workers
+          @workers.count { |worker| worker.alive? }
         end
 
-        def run
-          unless @options[:workers_only]
-            log_debug { sign_message("initializing broker") }
-            @broker = ::Protobuf::Rpc::Zmq::Broker.new(@options)
+        def backend_ip
+          frontend_ip
+        end
+
+        def backend_port
+          options[:worker_port]
+        end
+
+        def backend_uri
+          "tcp://#{backend_ip}:#{backend_port}"
+        end
+
+        def beacon_interval
+          [options[:beacon_interval].to_i, 1].max
+        end
+
+        def beacon_ip
+          "255.255.255.255"
+        end
+
+        def beacon_port
+          options[:beacon_port].to_i
+        end
+
+        def broadcast_beacons?
+          !brokerless? && options[:dynamic_discovery]
+        end
+
+        def broadcast_flatline
+          # TODO: create a flatline beacon from the proto
+          flatline = "flatline"
+
+          @beacon_socket.send flatline, 0
+        end
+
+        def broadcast_heartbeat
+          # TODO: create a heartbeat beacon from the proto
+          heartbeat = "heartbeat"
+
+          @beacon_socket.send heartbeat, 0
+        end
+
+        def brokerless?
+          !!options[:workers_only]
+        end
+
+        def frontend_ip
+          @frontend_ip ||= resolve_ip(options[:host])
+        end
+
+        def frontend_port
+          options[:port]
+        end
+
+        def frontend_uri
+          "tcp://#{frontend_ip}:#{frontend_port}"
+        end
+
+        def maintenance_interval
+          [reaping_interval, beacon_interval].min
+        end
+
+        def minimum_timeout
+          100
+        end
+
+        def reap_dead_workers
+          @workers.keep_if do |worker|
+            worker.alive?
           end
+        end
 
-          local_worker_threads = @options[:threads]
-          log_debug { sign_message("starting server workers") }
+        def reaping_interval
+          5
+        end
 
+        def run
           @running = true
 
-          local_worker_threads.times do
-            self.start_worker
-          end
+          start_missing_workers
 
-          log_debug { sign_message("server started") }
+          brokerless? ? wait_for_shutdown_signal : run_broker
 
-          while self.running? do
-            if @options[:workers_only]
-              sleep 5
-              Thread.pass
-            else
-              @broker.poll
-            end
+          @workers.each do |t|
+            t.join(5) || t.kill
           end
-        ensure
-          @broker.teardown if @broker
+
+          teardown
         end
 
         def running?
           !!@running
         end
 
+        def run_broker
+          log_debug { sign_message("initializing broker") }
+
+          timeout = 0
+          next_beacon = 0
+          next_reaping = 0
+          next_cycle = Time.now.to_i + maintenance_interval
+          poller = ZMQ::Poller.new
+          available_workers = []
+
+          poller.register_readable @frontend_socket
+          poller.register_readable @backend_socket
+          poller.register_readable @shutdown_socket
+
+          while running? || @workers.any?
+            poller.poll(timeout)
+            poller.readables.each do |readable|
+              if readable === @frontend_socket && available_workers.any?
+                @frontend_socket.recv_strings frames = []
+                @backend_socket.send_strings [available_workers.shift, ""] + frames
+              elsif readable === @backend_socket
+                @backend_socket.recv_strings frames = []
+                available_workers << frames.shift(2)[0]
+                unless frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
+                  @frontend_socket.send_strings frames
+                end
+              elsif readable === @shutdown_socket
+                broadcast_flatline if broadcast_beacons?
+                poller.deregister_readable @frontend_socket
+              end
+            end
+
+            if (time = Time.now.to_i) >= next_reaping
+              reap_dead_workers
+              start_missing_workers if running?
+              next_reaping = time + reaping_interval
+              next_cycle = [next_cycle, next_reaping].min
+            end
+
+            if broadcast_beacons? && time >= next_beacon
+              running? ? broadcast_heartbeat : broadcast_flatline
+              next_beacon = time + beacon_interval
+              next_cycle = [next_cycle, next_beacon].min
+            end
+
+            timeout = [minimum_timeout, 1_000 * (next_cycle - time)].max
+          end
+        end
+
+        def signal_shutdown
+          @running = false
+          socket = zmq_context.socket ZMQ::PAIR
+          zmq_error_check(socket.connect shutdown_uri)
+          zmq_error_check(socket.send_string "shutdown")
+          zmq_error_check(socket.close)
+        end
+
+        def start_missing_workers
+          missing_workers = total_workers - @workers.size
+
+          if missing_workers > 0
+            missing_workers.times { start_worker }
+            log_debug { sign_message("#{total_workers} workers started") }
+          end
+        end
+
         def start_worker
-          @threads << Thread.new(self) { |server|
+          @workers << Thread.new(self) do |server|
             begin
               ::Protobuf::Rpc::Zmq::Worker.new(server).run
             rescue => e
-              message =  "Worker Failed, spawning new worker: #{e.inspect}\n #{e.backtrace.join($/)}"
+              message = "Worker failed: #{e.inspect}\n #{e.backtrace.join($/)}"
               $stderr.puts message
               log_error { message }
-
-              retry if self.running?
             end
-          }
+          end
+        end
+
+        def shutdown_uri
+          "inproc://#{object_id}"
         end
 
         def stop
-          @running = false
+          signal_shutdown
+        end
 
-          @threads.each do |t|
-            t.join(5) || t.kill
-          end
+        def teardown
+          @frontend_socket.try :close
+          @backend_socket.try :close
+          @shutdown_socket.try :close
+          @beacon_socket.try :close
+          @zmq_context.terminate
+        end
+
+        def total_workers
+          @total_workers ||= @options[:threads]
+        end
+
+        def wait_for_shutdown_signal
+          @shutdown_socket.recv_string shutdown = ""
+        end
+
+        private
+
+        def default_options
+          { :beacon_interval => 5 }
+        end
+
+        def init_backend_socket
+          @backend_socket = @zmq_context.socket ZMQ::ROUTER
+          zmq_error_check(@backend_socket.bind backend_uri)
+        end
+
+        def init_beacon_socket
+          @beacon_socket = UDPSocket.new
+          @beacon_socket.setsockopt Socket::SOL_SOCKET, Socket::SO_BROADCAST, true
+          @beacon_socket.connect beacon_ip, beacon_port
+        end
+
+        def init_frontend_socket
+          @frontend_socket = @zmq_context.socket ZMQ::ROUTER
+          zmq_error_check(@frontend_socket.bind frontend_uri)
+        end
+
+        def init_shutdown_socket
+          @shutdown_socket = @zmq_context.socket ZMQ::PAIR
+          @shutdown_socket.bind shutdown_uri
         end
 
-        def worker_port
-          @options[:worker_port]
+        def init_zmq_context
+          @zmq_context = ZMQ::Context.new
         end
       end
     end
diff --git a/lib/protobuf/rpc/servers/zmq/util.rb b/lib/protobuf/rpc/servers/zmq/util.rb
index d8ff4508..0eba01f7 100644
--- a/lib/protobuf/rpc/servers/zmq/util.rb
+++ b/lib/protobuf/rpc/servers/zmq/util.rb
@@ -6,6 +6,7 @@ module Zmq
 
       module Util
         include ::Protobuf::Logger::LogMethods
+
         def self.included(base)
           base.extend(::Protobuf::Rpc::Zmq::Util)
         end
@@ -21,9 +22,7 @@ def log_signature
         def resolve_ip(hostname)
           ::Resolv.getaddress(hostname)
         end
-
       end
-
     end
   end
 end
diff --git a/lib/protobuf/rpc/servers/zmq/worker.rb b/lib/protobuf/rpc/servers/zmq/worker.rb
index 8ad18b18..c0eb7a42 100644
--- a/lib/protobuf/rpc/servers/zmq/worker.rb
+++ b/lib/protobuf/rpc/servers/zmq/worker.rb
@@ -1,9 +1,9 @@
 require 'protobuf/rpc/server'
 require 'protobuf/rpc/servers/zmq/util'
+
 module Protobuf
   module Rpc
     module Zmq
-
       class Worker
         include ::Protobuf::Rpc::Server
         include ::Protobuf::Rpc::Zmq::Util
@@ -13,18 +13,8 @@ class Worker
         #
         def initialize(server)
           @server = server
-          host = server.host
-          port = server.worker_port
-
-          @zmq_context = ::ZMQ::Context.new
-          @socket = @zmq_context.socket(::ZMQ::REQ)
-          zmq_error_check(@socket.connect("tcp://#{resolve_ip(host)}:#{port}"))
-
-          @poller = ::ZMQ::Poller.new
-          @poller.register(@socket, ::ZMQ::POLLIN)
-
-          # Send request to broker telling it we are ready
-          zmq_error_check(@socket.send_string(::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE))
+          init_zmq_context
+          init_socket
         end
 
         ##
@@ -42,11 +32,16 @@ def handle_request(socket)
         end
 
         def run
-          while @server.running? do
-            # poll for 1_000 milliseconds then continue looping
-            # This lets us see whether we need to die
-            @poller.poll(1_000)
-            @poller.readables.each do |socket|
+          poller = ::ZMQ::Poller.new
+          poller.register(@socket, ::ZMQ::POLLIN)
+
+          # Send request to broker telling it we are ready
+          zmq_error_check(@socket.send_string(::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE))
+
+          # poll for 1 second then continue looping
+          # This lets us see whether we need to die
+          while @server.running? && poller.poll(1_000) >= 0
+            poller.readables.each do |socket|
               initialize_request!
               handle_request(socket)
             end
@@ -68,8 +63,18 @@ def send_data
           @stats.response_size = response_data.size
           zmq_error_check(@socket.send_strings(response_message_set))
         end
-      end
 
+        private
+
+        def init_zmq_context
+          @zmq_context = ZMQ::Context.new
+        end
+
+        def init_socket
+          @socket = @zmq_context.socket ZMQ::REQ
+          zmq_error_check(@socket.connect @server.backend_uri)
+        end
+      end
     end
   end
 end
diff --git a/proto/dynamic_discovery.proto b/proto/dynamic_discovery.proto
new file mode 100644
index 00000000..81eb0fee
--- /dev/null
+++ b/proto/dynamic_discovery.proto
@@ -0,0 +1,40 @@
+// Copyright (c) 2013 MoneyDesktop, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+// Authors: Devin Christensen
+//
+// Protobufs needed for dynamic discovery zmq server and client.
+
+package protobuf.dynamic_discovery;
+
+enum BeaconType {
+  HEARTBEAT = 1;
+  FLATLINE = 2;
+}
+
+message Beacon {
+  optional BeaconType beacon_type = 1;
+  optional string uuid = 2;
+  optional string hostname = 3;
+  optional string port = 4;
+  optional int32 ttl = 5;
+  repeated string service_classes = 5;
+}
+
diff --git a/spec/lib/protobuf/rpc/servers/zmq/broker_spec.rb b/spec/lib/protobuf/rpc/servers/zmq/broker_spec.rb
deleted file mode 100644
index 6aef8433..00000000
--- a/spec/lib/protobuf/rpc/servers/zmq/broker_spec.rb
+++ /dev/null
@@ -1,31 +0,0 @@
-require 'spec_helper'
-
-describe ::Protobuf::Rpc::Zmq::Broker do
-  before(:each) do
-    load 'protobuf/zmq.rb'
-  end
-
-  after(:each) do
-    subject.teardown
-  end
-
-  subject do
-    described_class.new({ :host => '127.0.0.1', :port => 9399, :worker_port => 9400 })
-  end
-
-  it 'sets up a context' do
-    subject.context.should be_a(::ZMQ::Context)
-  end
-
-  it 'sets up a backend socket' do
-    subject.backend.should be_a(::ZMQ::Socket)
-  end
-
-  it 'sets up a polling object' do
-    subject.poller.should be_a(::ZMQ::Poller)
-  end
-
-  describe '#poll' do
-    # no unit tests for this method
-  end
-end
diff --git a/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb b/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb
index bf93393a..51db32d7 100644
--- a/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb
+++ b/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb
@@ -2,14 +2,32 @@
 require 'protobuf/rpc/servers/zmq/server'
 
 describe Protobuf::Rpc::Zmq::Server do
-  subject { described_class.new({}) }
+  subject { described_class.new(options) }
 
-  before(:each) do
+  let(:options) {{
+    :host => '127.0.0.1',
+    :port => 9399,
+    :worker_port => 9400,
+    :workers_only => true
+  }}
+
+  before do
     load 'protobuf/zmq.rb'
   end
 
-  describe '.running?' do
+  after do
+    subject.teardown
+  end
+
+  it 'sets up a ZMQ context' do
+    subject.instance_variable_get(:@zmq_context).should be_a(::ZMQ::Context)
+  end
 
+  it 'sets up a backend socket' do
+    subject.instance_variable_get(:@backend_socket).should be_a(::ZMQ::Socket)
+  end
+
+  describe '.running?' do
     it 'returns true if running' do
       subject.instance_variable_set(:@running, true)
       subject.running?.should be_true
@@ -22,21 +40,13 @@
   end
 
   describe '.stop' do
-    # keep threads instance variable from retaining any thread mocks we've
-    # created (breaks tests down the line, otherwise)
-    after(:each) do
-      subject.instance_variable_set(:@threads, [])
-    end
-
-    it 'lets all threads stop' do
-      thread_mock = double(Thread)
-      thread_mock.should_receive(:join).and_return(thread_mock)
-      subject.instance_variable_set(:@threads, [thread_mock])
+    it 'signals shutdown' do
+      subject.should_receive(:signal_shutdown)
       subject.stop
     end
 
     it 'sets running to false' do
-      subject.instance_variable_set(:@threads, [])
+      subject.instance_variable_set(:@workers, [])
       subject.stop
       subject.instance_variable_get(:@running).should be_false
     end

From 496d96ce37058635abc1830d5f1abc11e9053d3a Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 24 May 2013 16:21:32 -0600
Subject: [PATCH 14/70] Improve JRuby compatability and cleanup CLI

---
 lib/protobuf/cli.rb                           | 40 +++++++++++--------
 lib/protobuf/rpc/servers/socket/server.rb     |  4 +-
 spec/lib/protobuf/cli_spec.rb                 | 14 ++++---
 spec/lib/protobuf/enum_spec.rb                |  2 +-
 .../rpc/servers/socket_server_spec.rb         |  6 +--
 5 files changed, 37 insertions(+), 29 deletions(-)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index 809b0ffe..391be744 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -7,9 +7,7 @@
 
 module Protobuf
   class CLI < ::Thor
-    include ::Thor::Actions
-
-    attr_accessor :runner, :mode, :start_aborted
+    attr_accessor :runner, :mode
 
     default_task :start
 
@@ -38,7 +36,6 @@ class CLI < ::Thor
 
     def start(app_file)
       debug_say 'Configuring the rpc_server process'
-      @start_aborted = false
 
       configure_logger
       configure_traps
@@ -47,9 +44,9 @@ def start(app_file)
       configure_gc
       configure_deprecation_warnings
 
-      run_if_no_abort { require_application!(app_file) }
-      run_if_no_abort { configure_process_name(app_file) }
-      run_if_no_abort { start_server! }
+      require_application!(app_file) unless exit_requested?
+      configure_process_name(app_file) unless exit_requested?
+      start_server! unless exit_requested?
     rescue => e
       say_and_exit!('ERROR: RPC Server failed to start.', e)
     end
@@ -129,13 +126,16 @@ def configure_server_mode
       # TODO add signal handling for hot-reloading the application.
       def configure_traps
         debug_say 'Configuring traps'
-        [:INT, :QUIT, :TERM].each do |signal|
-          debug_say "Registering signal trap for #{signal}", :blue
+
+        exit_signals = [:INT, :TERM]
+        exit_signals << :QUIT unless defined?(JRUBY_VERSION)
+
+        exit_signals.each do |signal|
+          debug_say "Registering trap for exit signal #{signal}", :blue
+
           trap(signal) do
-            ::Protobuf::Logger.info { 'RPC Server shutting down...' }
-            @start_aborted = true
-            @runner.stop
-            ::Protobuf::Logger.info { 'Shutdown complete' }
+            @exit_requested = true
+            shutdown_server
           end
         end
       end
@@ -145,6 +145,10 @@ def debug_say(message, color = :yellow)
         say(message, color) if options.debug?
       end
 
+      def exit_requested?
+        !!@exit_requested
+      end
+
       # Internal helper to determine if the modes are multi-set which is not valid.
       def multi_mode?
         [
@@ -170,10 +174,6 @@ def require_protobuf!
         say_and_exit!("Failed to load protobuf runner #{@mode}", e)
       end
 
-      def run_if_no_abort
-        yield unless @start_aborted
-      end
-
       def runner_options
         {
           :host => options.host,
@@ -216,6 +216,12 @@ def server_zmq!
         @runner = ::Protobuf::Rpc::ZmqRunner.new(runner_options)
       end
 
+      def shutdown_server!
+        ::Protobuf::Logger.info { 'RPC Server shutting down...' }
+        @runner.try :stop
+        ::Protobuf::Logger.info { 'Shutdown complete' }
+      end
+
       # Start the runner and log the relevant options.
       def start_server!
         debug_say 'Invoking server start'
diff --git a/lib/protobuf/rpc/servers/socket/server.rb b/lib/protobuf/rpc/servers/socket/server.rb
index ed9ae980..b837a815 100644
--- a/lib/protobuf/rpc/servers/socket/server.rb
+++ b/lib/protobuf/rpc/servers/socket/server.rb
@@ -101,12 +101,12 @@ def run
         end
 
         def running?
-          @running
+          !!@running
         end
 
         def stop
           @running = false
-          @server.close if @server
+          @server.try :close
         end
       end
     end
diff --git a/spec/lib/protobuf/cli_spec.rb b/spec/lib/protobuf/cli_spec.rb
index 0de5e5ed..dd6d404d 100644
--- a/spec/lib/protobuf/cli_spec.rb
+++ b/spec/lib/protobuf/cli_spec.rb
@@ -96,16 +96,18 @@
 
         it 'sets both request and serialization pausing to false' do
           described_class.start(args)
-          ::Protobuf.gc_pause_server_request?.should be_false
+          ::Protobuf.should_not be_gc_pause_server_request
         end
       end
 
-      context 'request pausing' do
-        let(:test_args) { [ '--gc_pause_request' ] }
+      unless defined?(JRUBY_VERSION)
+        context 'request pausing' do
+          let(:test_args) { [ '--gc_pause_request' ] }
 
-        it 'sets the configuration option to GC pause server request' do
-          described_class.start(args)
-          ::Protobuf.gc_pause_server_request?.should be_true
+          it 'sets the configuration option to GC pause server request' do
+            described_class.start(args)
+            ::Protobuf.should be_gc_pause_server_request
+          end
         end
       end
     end
diff --git a/spec/lib/protobuf/enum_spec.rb b/spec/lib/protobuf/enum_spec.rb
index daa16dc0..afacc4ee 100644
--- a/spec/lib/protobuf/enum_spec.rb
+++ b/spec/lib/protobuf/enum_spec.rb
@@ -6,7 +6,7 @@
 
   before(:all) do
     Test::EnumTestType.define(:MINUS_ONE, -1)
-    Test::EnumTestType.define(name, tag)
+    Test::EnumTestType.define(:THREE, 3)
   end
 
   describe '.define' do
diff --git a/spec/lib/protobuf/rpc/servers/socket_server_spec.rb b/spec/lib/protobuf/rpc/servers/socket_server_spec.rb
index a0be83b7..84de8c13 100644
--- a/spec/lib/protobuf/rpc/servers/socket_server_spec.rb
+++ b/spec/lib/protobuf/rpc/servers/socket_server_spec.rb
@@ -25,11 +25,11 @@
   end
 
   it "Runner provides a stop method" do
-    @runner.respond_to?(:stop).should be_true
+    @runner.should respond_to(:stop)
   end
 
   it "provides a stop method" do
-    @runner.respond_to?(:stop).should be_true
+    @server.should respond_to(:stop)
   end
 
   it "provides a Runner class" do
@@ -38,7 +38,7 @@
   end
 
   it "signals the Server is running" do
-    @server.running?.should be_true
+    @server.should be_running
   end
 
 end

From e4950c868fdb2cea08faa288d5e6cb2ee53ca71f Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 24 May 2013 18:03:33 -0600
Subject: [PATCH 15/70] Integrate the beacon proto into the zmq server

---
 lib/protobuf/rpc/dynamic_discovery.pb.rb | 40 ++++++++++++++++++++
 lib/protobuf/rpc/rpc.pb.rb               | 25 +++++++------
 lib/protobuf/rpc/servers/zmq/server.rb   | 47 ++++++++++++++++--------
 lib/protobuf/rpc/servers/zmq/util.rb     |  2 +
 proto/dynamic_discovery.proto            | 10 ++---
 spec/functional/zmq_server_spec.rb       | 34 +++++++++--------
 6 files changed, 110 insertions(+), 48 deletions(-)
 create mode 100644 lib/protobuf/rpc/dynamic_discovery.pb.rb

diff --git a/lib/protobuf/rpc/dynamic_discovery.pb.rb b/lib/protobuf/rpc/dynamic_discovery.pb.rb
new file mode 100644
index 00000000..056ec020
--- /dev/null
+++ b/lib/protobuf/rpc/dynamic_discovery.pb.rb
@@ -0,0 +1,40 @@
+##
+# This file is auto-generated. DO NOT EDIT!
+#
+require 'protobuf/message'
+
+module Protobuf
+
+  module DynamicDiscovery
+  
+    ##
+    # Enum Classes
+    #
+    class BeaconType < ::Protobuf::Enum
+      define :HEARTBEAT, 0
+      define :FLATLINE, 1
+    end
+    
+    
+    ##
+    # Message Classes
+    #
+    class Beacon < ::Protobuf::Message; end
+    
+    ##
+    # Message Fields
+    #
+    class Beacon
+      optional ::Protobuf::DynamicDiscovery::BeaconType, :beacon_type, 1
+      optional ::Protobuf::Field::StringField, :uuid, 2
+      optional ::Protobuf::Field::StringField, :address, 3
+      optional ::Protobuf::Field::StringField, :port, 4
+      optional ::Protobuf::Field::Int32Field, :ttl, 5
+      repeated ::Protobuf::Field::StringField, :service_classes, 6
+    end
+    
+    
+  end
+  
+end
+
diff --git a/lib/protobuf/rpc/rpc.pb.rb b/lib/protobuf/rpc/rpc.pb.rb
index d7b12fca..fc3d69bb 100644
--- a/lib/protobuf/rpc/rpc.pb.rb
+++ b/lib/protobuf/rpc/rpc.pb.rb
@@ -4,22 +4,13 @@
 require 'protobuf/message'
 
 module Protobuf
+
   module Socketrpc
+  
     ##
     # Enum Classes
     #
-    class ErrorReason < ::Protobuf::Enum; end
-    
-    ##
-    # Message Classes
-    #
-    class Request < ::Protobuf::Message; end
-    class Response < ::Protobuf::Message; end
-    
-    ##
-    # Enum Values
-    #
-    class ErrorReason
+    class ErrorReason < ::Protobuf::Enum
       define :BAD_REQUEST_DATA, 0
       define :BAD_REQUEST_PROTO, 1
       define :SERVICE_NOT_FOUND, 2
@@ -32,6 +23,13 @@ class ErrorReason
       define :IO_ERROR, 9
     end
     
+    
+    ##
+    # Message Classes
+    #
+    class Request < ::Protobuf::Message; end
+    class Response < ::Protobuf::Message; end
+    
     ##
     # Message Fields
     #
@@ -49,5 +47,8 @@ class Response
       optional ::Protobuf::Socketrpc::ErrorReason, :error_reason, 4
     end
     
+    
   end
+  
 end
+
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index c410e90c..d15da2c4 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -1,5 +1,7 @@
 require 'protobuf/rpc/servers/zmq/worker'
 require 'protobuf/rpc/servers/zmq/util'
+require 'protobuf/rpc/dynamic_discovery.pb'
+require 'securerandom'
 
 module Protobuf
   module Rpc
@@ -20,10 +22,6 @@ def initialize(options)
           init_beacon_socket if broadcast_beacons?
         end
 
-        def alive_workers
-          @workers.count { |worker| worker.alive? }
-        end
-
         def backend_ip
           frontend_ip
         end
@@ -53,17 +51,27 @@ def broadcast_beacons?
         end
 
         def broadcast_flatline
-          # TODO: create a flatline beacon from the proto
-          flatline = "flatline"
+          flatline = ::Protobuf::DynamicDiscovery::Beacon.new(
+            :beacon_type => ::Protobuf::DynamicDiscovery::BeaconType::FLATLINE,
+            :uuid => uuid,
+            :address => frontend_ip,
+            :port => frontend_port.to_s
+          )
 
-          @beacon_socket.send flatline, 0
+          @beacon_socket.send flatline.to_s, 0
         end
 
         def broadcast_heartbeat
-          # TODO: create a heartbeat beacon from the proto
-          heartbeat = "heartbeat"
+          heartbeat = ::Protobuf::DynamicDiscovery::Beacon.new(
+            :beacon_type => ::Protobuf::DynamicDiscovery::BeaconType::HEARTBEAT,
+            :uuid => uuid,
+            :address => frontend_ip,
+            :port => frontend_port.to_s,
+            :ttl => beacon_interval * 3,
+            :service_classes => ["not sure"] # TODO: Figure out what goes here
+          )
 
-          @beacon_socket.send heartbeat, 0
+          @beacon_socket.send heartbeat.to_s, 0
         end
 
         def brokerless?
@@ -110,7 +118,7 @@ def run
           @workers.each do |t|
             t.join(5) || t.kill
           end
-
+        ensure
           teardown
         end
 
@@ -135,9 +143,11 @@ def run_broker
           while running? || @workers.any?
             poller.poll(timeout)
             poller.readables.each do |readable|
-              if readable === @frontend_socket && available_workers.any?
-                @frontend_socket.recv_strings frames = []
-                @backend_socket.send_strings [available_workers.shift, ""] + frames
+              if readable === @frontend_socket
+                if available_workers.any?
+                  @frontend_socket.recv_strings frames = []
+                  @backend_socket.send_strings [available_workers.shift, ""] + frames
+                end
               elsif readable === @backend_socket
                 @backend_socket.recv_strings frames = []
                 available_workers << frames.shift(2)[0]
@@ -147,6 +157,7 @@ def run_broker
               elsif readable === @shutdown_socket
                 broadcast_flatline if broadcast_beacons?
                 poller.deregister_readable @frontend_socket
+                poller.deregister_readable @shutdown_socket
               end
             end
 
@@ -216,6 +227,10 @@ def total_workers
           @total_workers ||= @options[:threads]
         end
 
+        def uuid
+          @uuid ||= SecureRandom.uuid
+        end
+
         def wait_for_shutdown_signal
           @shutdown_socket.recv_string shutdown = ""
         end
@@ -223,7 +238,7 @@ def wait_for_shutdown_signal
         private
 
         def default_options
-          { :beacon_interval => 5 }
+          { :beacon_interval => 5  }
         end
 
         def init_backend_socket
@@ -233,7 +248,7 @@ def init_backend_socket
 
         def init_beacon_socket
           @beacon_socket = UDPSocket.new
-          @beacon_socket.setsockopt Socket::SOL_SOCKET, Socket::SO_BROADCAST, true
+          @beacon_socket.setsockopt :SOCKET, :BROADCAST, true
           @beacon_socket.connect beacon_ip, beacon_port
         end
 
diff --git a/lib/protobuf/rpc/servers/zmq/util.rb b/lib/protobuf/rpc/servers/zmq/util.rb
index 0eba01f7..078de305 100644
--- a/lib/protobuf/rpc/servers/zmq/util.rb
+++ b/lib/protobuf/rpc/servers/zmq/util.rb
@@ -1,3 +1,5 @@
+require 'resolv'
+
 module Protobuf
   module Rpc
     module Zmq
diff --git a/proto/dynamic_discovery.proto b/proto/dynamic_discovery.proto
index 81eb0fee..3854a105 100644
--- a/proto/dynamic_discovery.proto
+++ b/proto/dynamic_discovery.proto
@@ -22,19 +22,19 @@
 //
 // Protobufs needed for dynamic discovery zmq server and client.
 
-package protobuf.dynamic_discovery;
+package protobuf.dynamicDiscovery;
 
 enum BeaconType {
-  HEARTBEAT = 1;
-  FLATLINE = 2;
+  HEARTBEAT = 0;
+  FLATLINE = 1;
 }
 
 message Beacon {
   optional BeaconType beacon_type = 1;
   optional string uuid = 2;
-  optional string hostname = 3;
+  optional string address = 3;
   optional string port = 4;
   optional int32 ttl = 5;
-  repeated string service_classes = 5;
+  repeated string service_classes = 6;
 }
 
diff --git a/spec/functional/zmq_server_spec.rb b/spec/functional/zmq_server_spec.rb
index 302e3eda..752fd4b6 100644
--- a/spec/functional/zmq_server_spec.rb
+++ b/spec/functional/zmq_server_spec.rb
@@ -2,25 +2,29 @@
 require 'spec/support/test/resource_service'
 
 describe 'Functional ZMQ Client' do
-  before(:all) do
-    load "protobuf/zmq.rb"
-    Thread.abort_on_exception = true
-    options = OpenStruct.new(:host => "127.0.0.1",
-                             :port => 9399,
-                             :worker_port => 9400,
-                             :backlog => 100,
-                             :threshold => 100,
-                             :threads => 5)
+  let(:options) {{
+    :host => "127.0.0.1",
+    :port => 9399,
+    :worker_port => 9400,
+    :backlog => 100,
+    :threshold => 100,
+    :threads => 5,
+    :dynamic_discovery => true,
+    :beacon_port => 9398
+  }}
 
-    @runner = ::Protobuf::Rpc::ZmqRunner.new(options)
-    @server_thread = Thread.new(@runner) { |runner| runner.run }
+  let(:server) { ::Protobuf::Rpc::Zmq::Server.new(options) }
+  let(:server_thread) { Thread.new(server) { |server| server.run } }
 
-    Thread.pass until @runner.running?
+  before do
+    load "protobuf/zmq.rb"
+    server_thread.abort_on_exception = true
+    Thread.pass until server.running?
   end
 
-  after(:all) do
-    @runner.stop
-    @server_thread.try(:join)
+  after do
+    server.stop
+    server_thread.join
   end
 
   it 'runs fine when required fields are set' do

From acaade92098d7d42c3506581f00fcc18774b1a16 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Tue, 28 May 2013 16:33:57 -0600
Subject: [PATCH 16/70] Separate the broker concern

---
 lib/protobuf/rpc/servers/zmq/broker.rb        | 114 +++++++++++++
 lib/protobuf/rpc/servers/zmq/server.rb        | 153 +++++++-----------
 lib/protobuf/rpc/servers/zmq/util.rb          |   3 +-
 lib/protobuf/rpc/servers/zmq/worker.rb        |  84 +++++++---
 .../protobuf/rpc/servers/zmq/server_spec.rb   |   8 -
 5 files changed, 241 insertions(+), 121 deletions(-)
 create mode 100644 lib/protobuf/rpc/servers/zmq/broker.rb

diff --git a/lib/protobuf/rpc/servers/zmq/broker.rb b/lib/protobuf/rpc/servers/zmq/broker.rb
new file mode 100644
index 00000000..3b5d1e27
--- /dev/null
+++ b/lib/protobuf/rpc/servers/zmq/broker.rb
@@ -0,0 +1,114 @@
+
+module Protobuf
+  module Rpc
+    module Zmq
+      class Broker
+        include ::Protobuf::Rpc::Zmq::Util
+
+        attr_reader :server
+
+        def initialize(server)
+          @server = server
+
+          init_zmq_context
+          init_backend_socket
+          init_frontend_socket
+          init_shutdown_socket
+          init_poller
+        end
+
+        def join
+          @thread.try :join
+        end
+
+        def minimum_timeout
+          100
+        end
+
+        def run
+          idle_workers = []
+
+          catch(:shutdown) do
+            while @poller.poll > 0
+              @poller.readables.each do |readable|
+                case readable
+                when @frontend_socket
+                  if idle_workers.any?
+                    @frontend_socket.recv_strings frames = []
+                    @backend_socket.send_strings [idle_workers.shift, ""] + frames
+                  end
+                when @backend_socket
+                  @backend_socket.recv_strings frames = []
+                  idle_workers << frames.shift(2)[0]
+                  unless frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
+                    @frontend_socket.send_strings frames
+                  end
+                when @shutdown_socket
+                  throw :shutdown
+                end
+              end
+            end
+          end
+
+          teardown
+        end
+
+        def start
+          log_debug { sign_message("starting broker") }
+
+          @thread = Thread.new do
+            self.run
+          end
+
+          self
+        end
+
+        def shutdown_uri
+          "inproc://#{object_id}"
+        end
+
+        def signal_shutdown
+          socket = @zmq_context.socket ZMQ::PAIR
+          zmq_error_check(socket.connect shutdown_uri)
+          zmq_error_check(socket.send_string "")
+          zmq_error_check(socket.close)
+        end
+
+        def teardown
+          @frontend_socket.close
+          @backend_socket.close
+          @shutdown_socket.close
+          @zmq_context.terminate
+        end
+
+        private
+
+        def init_backend_socket
+          @backend_socket = @zmq_context.socket ZMQ::ROUTER
+          zmq_error_check(@backend_socket.bind server.backend_uri)
+        end
+
+        def init_frontend_socket
+          @frontend_socket = @zmq_context.socket ZMQ::ROUTER
+          zmq_error_check(@frontend_socket.bind server.frontend_uri)
+        end
+
+        def init_poller
+          @poller = ZMQ::Poller.new
+          @poller.register_readable @frontend_socket
+          @poller.register_readable @backend_socket
+          @poller.register_readable @shutdown_socket
+        end
+
+        def init_shutdown_socket
+          @shutdown_socket = @zmq_context.socket ZMQ::PAIR
+          zmq_error_check(@shutdown_socket.bind shutdown_uri)
+        end
+
+        def init_zmq_context
+          @zmq_context = ZMQ::Context.new
+        end
+      end
+    end
+  end
+end
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index d15da2c4..a7fc3b32 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -1,8 +1,11 @@
-require 'protobuf/rpc/servers/zmq/worker'
 require 'protobuf/rpc/servers/zmq/util'
+require 'protobuf/rpc/servers/zmq/worker'
+require 'protobuf/rpc/servers/zmq/broker'
 require 'protobuf/rpc/dynamic_discovery.pb'
 require 'securerandom'
 
+STDOUT.sync = true
+
 module Protobuf
   module Rpc
     module Zmq
@@ -16,10 +19,11 @@ def initialize(options)
           @workers = []
 
           init_zmq_context
-          init_backend_socket
-          init_frontend_socket unless brokerless?
-          init_shutdown_socket
           init_beacon_socket if broadcast_beacons?
+          init_shutdown_socket
+        rescue
+          teardown
+          raise
         end
 
         def backend_ip
@@ -100,7 +104,7 @@ def minimum_timeout
 
         def reap_dead_workers
           @workers.keep_if do |worker|
-            worker.alive?
+            worker.thread.alive? or worker.thread.join && false
           end
         end
 
@@ -111,14 +115,13 @@ def reaping_interval
         def run
           @running = true
 
+          start_broker unless brokerless?
           start_missing_workers
-
-          brokerless? ? wait_for_shutdown_signal : run_broker
-
-          @workers.each do |t|
-            t.join(5) || t.kill
-          end
+          wait_for_shutdown_signal
+          stop_workers
+          stop_broker unless brokerless?
         ensure
+          @running = false
           teardown
         end
 
@@ -126,66 +129,21 @@ def running?
           !!@running
         end
 
-        def run_broker
-          log_debug { sign_message("initializing broker") }
-
-          timeout = 0
-          next_beacon = 0
-          next_reaping = 0
-          next_cycle = Time.now.to_i + maintenance_interval
-          poller = ZMQ::Poller.new
-          available_workers = []
-
-          poller.register_readable @frontend_socket
-          poller.register_readable @backend_socket
-          poller.register_readable @shutdown_socket
-
-          while running? || @workers.any?
-            poller.poll(timeout)
-            poller.readables.each do |readable|
-              if readable === @frontend_socket
-                if available_workers.any?
-                  @frontend_socket.recv_strings frames = []
-                  @backend_socket.send_strings [available_workers.shift, ""] + frames
-                end
-              elsif readable === @backend_socket
-                @backend_socket.recv_strings frames = []
-                available_workers << frames.shift(2)[0]
-                unless frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
-                  @frontend_socket.send_strings frames
-                end
-              elsif readable === @shutdown_socket
-                broadcast_flatline if broadcast_beacons?
-                poller.deregister_readable @frontend_socket
-                poller.deregister_readable @shutdown_socket
-              end
-            end
-
-            if (time = Time.now.to_i) >= next_reaping
-              reap_dead_workers
-              start_missing_workers if running?
-              next_reaping = time + reaping_interval
-              next_cycle = [next_cycle, next_reaping].min
-            end
-
-            if broadcast_beacons? && time >= next_beacon
-              running? ? broadcast_heartbeat : broadcast_flatline
-              next_beacon = time + beacon_interval
-              next_cycle = [next_cycle, next_beacon].min
-            end
-
-            timeout = [minimum_timeout, 1_000 * (next_cycle - time)].max
-          end
+        def shutdown_uri
+          "inproc://#{object_id}"
         end
 
         def signal_shutdown
-          @running = false
-          socket = zmq_context.socket ZMQ::PAIR
+          socket = @zmq_context.socket ZMQ::PAIR
           zmq_error_check(socket.connect shutdown_uri)
-          zmq_error_check(socket.send_string "shutdown")
+          zmq_error_check(socket.send_string "blargh!")
           zmq_error_check(socket.close)
         end
 
+        def start_broker
+          @broker = ::Protobuf::Rpc::Zmq::Broker.new(self).start
+        end
+
         def start_missing_workers
           missing_workers = total_workers - @workers.size
 
@@ -196,31 +154,27 @@ def start_missing_workers
         end
 
         def start_worker
-          @workers << Thread.new(self) do |server|
-            begin
-              ::Protobuf::Rpc::Zmq::Worker.new(server).run
-            rescue => e
-              message = "Worker failed: #{e.inspect}\n #{e.backtrace.join($/)}"
-              $stderr.puts message
-              log_error { message }
-            end
-          end
-        end
-
-        def shutdown_uri
-          "inproc://#{object_id}"
+          @workers << ::Protobuf::Rpc::Zmq::Worker.new(self).start
         end
 
         def stop
           signal_shutdown
         end
 
+        def stop_broker
+          @broker.signal_shutdown
+          @broker.join
+        end
+
+        def stop_workers
+          @workers.each &:signal_shutdown
+          Thread.pass until reap_dead_workers.empty?
+        end
+
         def teardown
-          @frontend_socket.try :close
-          @backend_socket.try :close
           @shutdown_socket.try :close
           @beacon_socket.try :close
-          @zmq_context.terminate
+          @zmq_context.try :terminate
         end
 
         def total_workers
@@ -232,7 +186,34 @@ def uuid
         end
 
         def wait_for_shutdown_signal
-          @shutdown_socket.recv_string shutdown = ""
+          timeout = 0
+          next_beacon = 0
+          next_reaping = 0
+          next_cycle = Time.now.to_i + maintenance_interval
+          poller = ZMQ::Poller.new
+
+          poller.register_readable @shutdown_socket
+
+          while poller.poll(timeout) >= 0
+            break if poller.readables.any?
+
+            time = Time.now.to_i
+
+            if time >= next_reaping
+              reap_dead_workers
+              start_missing_workers
+              next_reaping = time + reaping_interval
+              next_cycle = [next_cycle, next_reaping].min
+            end
+
+            if broadcast_beacons? && time >= next_beacon
+              broadcast_heartbeat
+              next_beacon = time + beacon_interval
+              next_cycle = [next_cycle, next_beacon].min
+            end
+
+            timeout = [minimum_timeout, 1000 * (next_cycle - time)].max
+          end
         end
 
         private
@@ -241,22 +222,12 @@ def default_options
           { :beacon_interval => 5  }
         end
 
-        def init_backend_socket
-          @backend_socket = @zmq_context.socket ZMQ::ROUTER
-          zmq_error_check(@backend_socket.bind backend_uri)
-        end
-
         def init_beacon_socket
           @beacon_socket = UDPSocket.new
           @beacon_socket.setsockopt :SOCKET, :BROADCAST, true
           @beacon_socket.connect beacon_ip, beacon_port
         end
 
-        def init_frontend_socket
-          @frontend_socket = @zmq_context.socket ZMQ::ROUTER
-          zmq_error_check(@frontend_socket.bind frontend_uri)
-        end
-
         def init_shutdown_socket
           @shutdown_socket = @zmq_context.socket ZMQ::PAIR
           @shutdown_socket.bind shutdown_uri
diff --git a/lib/protobuf/rpc/servers/zmq/util.rb b/lib/protobuf/rpc/servers/zmq/util.rb
index 078de305..513eaa45 100644
--- a/lib/protobuf/rpc/servers/zmq/util.rb
+++ b/lib/protobuf/rpc/servers/zmq/util.rb
@@ -4,7 +4,8 @@ module Protobuf
   module Rpc
     module Zmq
 
-      WORKER_READY_MESSAGE = "WORKER_READY"
+      WORKER_READY_MESSAGE = "\1"
+      WORKER_SHUTDOWN_MESSAGE = "\2"
 
       module Util
         include ::Protobuf::Logger::LogMethods
diff --git a/lib/protobuf/rpc/servers/zmq/worker.rb b/lib/protobuf/rpc/servers/zmq/worker.rb
index c0eb7a42..88f0e133 100644
--- a/lib/protobuf/rpc/servers/zmq/worker.rb
+++ b/lib/protobuf/rpc/servers/zmq/worker.rb
@@ -8,13 +8,16 @@ class Worker
         include ::Protobuf::Rpc::Server
         include ::Protobuf::Rpc::Zmq::Util
 
+        attr_reader :thread
+
         ##
         # Constructor
         #
         def initialize(server)
           @server = server
           init_zmq_context
-          init_socket
+          init_backend_socket
+          init_shutdown_socket
         end
 
         ##
@@ -32,36 +35,70 @@ def handle_request(socket)
         end
 
         def run
+          running = true
           poller = ::ZMQ::Poller.new
-          poller.register(@socket, ::ZMQ::POLLIN)
+
+          poller.register_readable(@backend_socket)
+          poller.register_readable(@shutdown_socket)
 
           # Send request to broker telling it we are ready
-          zmq_error_check(@socket.send_string(::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE))
-
-          # poll for 1 second then continue looping
-          # This lets us see whether we need to die
-          while @server.running? && poller.poll(1_000) >= 0
-            poller.readables.each do |socket|
-              initialize_request!
-              handle_request(socket)
+          zmq_error_check(@backend_socket.send_string(::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE))
+
+          while poller.poll > 0
+            poller.readables.each do |readable|
+              if readable === @backend_socket
+                initialize_request!
+                handle_request(@backend_socket)
+              elsif readable === @shutdown_socket
+                running = false
+              end
             end
+
+            break unless running
           end
         ensure
-          @socket.close
-          @zmq_context.terminate
+          teardown
         end
 
         def send_data
           response_data = @response.to_s # to_s is aliases as serialize_to_string in Message
 
-          response_message_set = [
-            @client_address, # client uuid address
-            "",
-            response_data
-          ]
+          frames = [@client_address, "", response_data]
 
           @stats.response_size = response_data.size
-          zmq_error_check(@socket.send_strings(response_message_set))
+
+          zmq_error_check(@backend_socket.send_strings(frames))
+        end
+
+        def shutdown_uri
+          "inproc://#{object_id}"
+        end
+
+        def signal_shutdown
+          socket = @zmq_context.socket ZMQ::PAIR
+          zmq_error_check(socket.connect shutdown_uri)
+          zmq_error_check(socket.send_string "shutdown")
+          zmq_error_check(socket.close)
+        end
+
+        def start
+          @thread = Thread.new do
+            begin
+              self.run
+            rescue => e
+              message = "Worker failed: #{e.inspect}\n #{e.backtrace.join($/)}"
+              $stderr.puts message
+              log_error { message }
+            end
+          end
+
+          self
+        end
+
+        def teardown
+          @backend_socket.try :close
+          @shutdown_socket.try :close
+          @zmq_context.try :terminate
         end
 
         private
@@ -70,9 +107,14 @@ def init_zmq_context
           @zmq_context = ZMQ::Context.new
         end
 
-        def init_socket
-          @socket = @zmq_context.socket ZMQ::REQ
-          zmq_error_check(@socket.connect @server.backend_uri)
+        def init_backend_socket
+          @backend_socket = @zmq_context.socket ZMQ::REQ
+          zmq_error_check(@backend_socket.connect @server.backend_uri)
+        end
+
+        def init_shutdown_socket
+          @shutdown_socket = @zmq_context.socket ZMQ::PAIR
+          zmq_error_check(@shutdown_socket.bind shutdown_uri)
         end
       end
     end
diff --git a/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb b/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb
index 51db32d7..80ab752e 100644
--- a/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb
+++ b/spec/lib/protobuf/rpc/servers/zmq/server_spec.rb
@@ -19,14 +19,6 @@
     subject.teardown
   end
 
-  it 'sets up a ZMQ context' do
-    subject.instance_variable_get(:@zmq_context).should be_a(::ZMQ::Context)
-  end
-
-  it 'sets up a backend socket' do
-    subject.instance_variable_get(:@backend_socket).should be_a(::ZMQ::Socket)
-  end
-
   describe '.running?' do
     it 'returns true if running' do
       subject.instance_variable_set(:@running, true)

From 1a3eb490b3b93e18d801647607e2ed2e012de944 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 11:20:42 -0600
Subject: [PATCH 17/70] Track subclasses and services

---
 lib/protobuf/rpc/service.rb | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/lib/protobuf/rpc/service.rb b/lib/protobuf/rpc/service.rb
index 86320a4c..4246fbd0 100644
--- a/lib/protobuf/rpc/service.rb
+++ b/lib/protobuf/rpc/service.rb
@@ -54,6 +54,11 @@ def self.host=(new_host)
         @_host = new_host
       end
 
+      def self.inherited(subclass)
+        @_subclasses ||= []
+        @_subclasses << subclass
+      end
+
       # Shorthand call to configure, passing a string formatted as hostname:port
       # e.g. 127.0.0.1:9933
       # e.g. localhost:0
@@ -96,6 +101,15 @@ def self.rpc_method?(name)
         rpcs.key?(name)
       end
 
+      # An array of defined service classes
+      def self.services
+        @_subclasses.select do |subclass|
+          subclass.rpcs.any? do |(name, method)|
+            subclass.method_defined? name
+          end
+        end
+      end
+
       ##
       # Instance Methods
       #

From a4dc84b681538c8d94f2eee30404f012c2a97448 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 11:21:45 -0600
Subject: [PATCH 18/70] Add services to the heartbeat beacon

---
 lib/protobuf/rpc/servers/zmq/server.rb | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index a7fc3b32..ba004967 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -66,13 +66,14 @@ def broadcast_flatline
         end
 
         def broadcast_heartbeat
+          services = ::Protobuf::Rpc::Service.services
           heartbeat = ::Protobuf::DynamicDiscovery::Beacon.new(
             :beacon_type => ::Protobuf::DynamicDiscovery::BeaconType::HEARTBEAT,
             :uuid => uuid,
             :address => frontend_ip,
             :port => frontend_port.to_s,
             :ttl => beacon_interval * 3,
-            :service_classes => ["not sure"] # TODO: Figure out what goes here
+            :service_classes => services.map(&:name)
           )
 
           @beacon_socket.send heartbeat.to_s, 0

From cc22b9cf318e79205f59d40a8afd669c97ab8651 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 11:22:43 -0600
Subject: [PATCH 19/70] Delegate alive? and join to the worker's thread

---
 lib/protobuf/rpc/servers/zmq/server.rb | 2 +-
 lib/protobuf/rpc/servers/zmq/worker.rb | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index ba004967..c85f991e 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -105,7 +105,7 @@ def minimum_timeout
 
         def reap_dead_workers
           @workers.keep_if do |worker|
-            worker.thread.alive? or worker.thread.join && false
+            worker.alive? or worker.join && false
           end
         end
 
diff --git a/lib/protobuf/rpc/servers/zmq/worker.rb b/lib/protobuf/rpc/servers/zmq/worker.rb
index 88f0e133..34c5f52f 100644
--- a/lib/protobuf/rpc/servers/zmq/worker.rb
+++ b/lib/protobuf/rpc/servers/zmq/worker.rb
@@ -23,6 +23,10 @@ def initialize(server)
         ##
         # Instance Methods
         #
+        def alive?
+          @thread.try(:alive?) || false
+        end
+
         def handle_request(socket)
           zmq_error_check(socket.recv_strings(frames = []))
 
@@ -34,6 +38,10 @@ def handle_request(socket)
           end
         end
 
+        def join
+          @thread.try :join
+        end
+
         def run
           running = true
           poller = ::ZMQ::Poller.new

From 23fb0a11936d55ef8a62099c876c7a7226ca026a Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 11:23:32 -0600
Subject: [PATCH 20/70] Broadcast the flatline beacon after shutdown

---
 lib/protobuf/rpc/servers/zmq/server.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index c85f991e..583e5968 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -119,6 +119,7 @@ def run
           start_broker unless brokerless?
           start_missing_workers
           wait_for_shutdown_signal
+          broadcast_flatline if broadcast_beacons?
           stop_workers
           stop_broker unless brokerless?
         ensure

From 78f979107bad910e20d25fc829f89eee76d973ef Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 11:24:23 -0600
Subject: [PATCH 21/70] Avoid reaping workers immediately after startup

---
 lib/protobuf/rpc/servers/zmq/server.rb | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 583e5968..4f83dd2b 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -188,10 +188,11 @@ def uuid
         end
 
         def wait_for_shutdown_signal
+          time = Time.now.to_i
           timeout = 0
           next_beacon = 0
-          next_reaping = 0
-          next_cycle = Time.now.to_i + maintenance_interval
+          next_reaping = time + reaping_interval
+          next_cycle = time + maintenance_interval
           poller = ZMQ::Poller.new
 
           poller.register_readable @shutdown_socket

From 11d93e719d51cd5ac59fe5e1252903a8a51937da Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 11:37:39 -0600
Subject: [PATCH 22/70] Tweak the poller loop to be a bit more concise

---
 lib/protobuf/rpc/servers/zmq/server.rb | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 4f83dd2b..1f80985c 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -194,15 +194,12 @@ def wait_for_shutdown_signal
           next_reaping = time + reaping_interval
           next_cycle = time + maintenance_interval
           poller = ZMQ::Poller.new
-
           poller.register_readable @shutdown_socket
 
-          while poller.poll(timeout) >= 0
-            break if poller.readables.any?
-
-            time = Time.now.to_i
-
-            if time >= next_reaping
+          # If the poller returns 1, a shutdown signal has been received.
+          # If the poller returns -1, something went wrong.
+          while poller.poll(timeout) === 0
+            if (time = Time.now.to_i) >= next_reaping
               reap_dead_workers
               start_missing_workers
               next_reaping = time + reaping_interval

From 6b94e796661c43c9b3bcce5ed4ad3c855e2de21f Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 11:43:54 -0600
Subject: [PATCH 23/70] Remove unused method in broker

---
 lib/protobuf/rpc/servers/zmq/broker.rb | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq/broker.rb b/lib/protobuf/rpc/servers/zmq/broker.rb
index 3b5d1e27..337d2c6e 100644
--- a/lib/protobuf/rpc/servers/zmq/broker.rb
+++ b/lib/protobuf/rpc/servers/zmq/broker.rb
@@ -21,10 +21,6 @@ def join
           @thread.try :join
         end
 
-        def minimum_timeout
-          100
-        end
-
         def run
           idle_workers = []
 

From 3f90238798fc4542c6a4b518f99b0e3a7aa2b31f Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 13:48:58 -0600
Subject: [PATCH 24/70] Review and improve code

---
 lib/protobuf/rpc/servers/zmq/broker.rb | 30 ++++++++++---------
 lib/protobuf/rpc/servers/zmq/server.rb | 10 +++----
 lib/protobuf/rpc/servers/zmq/util.rb   |  1 -
 lib/protobuf/rpc/servers/zmq/worker.rb | 40 +++++++++++++-------------
 lib/protobuf/rpc/service.rb            |  2 +-
 5 files changed, 41 insertions(+), 42 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq/broker.rb b/lib/protobuf/rpc/servers/zmq/broker.rb
index 337d2c6e..2aae9006 100644
--- a/lib/protobuf/rpc/servers/zmq/broker.rb
+++ b/lib/protobuf/rpc/servers/zmq/broker.rb
@@ -5,8 +5,6 @@ module Zmq
       class Broker
         include ::Protobuf::Rpc::Zmq::Util
 
-        attr_reader :server
-
         def initialize(server)
           @server = server
 
@@ -15,6 +13,9 @@ def initialize(server)
           init_frontend_socket
           init_shutdown_socket
           init_poller
+        rescue
+          teardown
+          raise
         end
 
         def join
@@ -30,14 +31,15 @@ def run
                 case readable
                 when @frontend_socket
                   if idle_workers.any?
-                    @frontend_socket.recv_strings frames = []
-                    @backend_socket.send_strings [idle_workers.shift, ""] + frames
+                    zmq_error_check(@frontend_socket.recv_strings frames = [])
+                    frames.unshift(idle_workers.shift, "")
+                    zmq_error_check(@backend_socket.send_strings frames)
                   end
                 when @backend_socket
-                  @backend_socket.recv_strings frames = []
+                  zmq_error_check(@backend_socket.recv_strings frames = [])
                   idle_workers << frames.shift(2)[0]
                   unless frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
-                    @frontend_socket.send_strings frames
+                    zmq_error_check(@frontend_socket.send_strings frames)
                   end
                 when @shutdown_socket
                   throw :shutdown
@@ -45,7 +47,7 @@ def run
               end
             end
           end
-
+        ensure
           teardown
         end
 
@@ -66,27 +68,27 @@ def shutdown_uri
         def signal_shutdown
           socket = @zmq_context.socket ZMQ::PAIR
           zmq_error_check(socket.connect shutdown_uri)
-          zmq_error_check(socket.send_string "")
+          zmq_error_check(socket.send_string ".")
           zmq_error_check(socket.close)
         end
 
         def teardown
-          @frontend_socket.close
-          @backend_socket.close
-          @shutdown_socket.close
-          @zmq_context.terminate
+          @frontend_socket.try :close
+          @backend_socket.try :close
+          @shutdown_socket.try :close
+          @zmq_context.try :terminate
         end
 
         private
 
         def init_backend_socket
           @backend_socket = @zmq_context.socket ZMQ::ROUTER
-          zmq_error_check(@backend_socket.bind server.backend_uri)
+          zmq_error_check(@backend_socket.bind @server.backend_uri)
         end
 
         def init_frontend_socket
           @frontend_socket = @zmq_context.socket ZMQ::ROUTER
-          zmq_error_check(@frontend_socket.bind server.frontend_uri)
+          zmq_error_check(@frontend_socket.bind @server.frontend_uri)
         end
 
         def init_poller
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 1f80985c..6c773692 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -4,15 +4,13 @@
 require 'protobuf/rpc/dynamic_discovery.pb'
 require 'securerandom'
 
-STDOUT.sync = true
-
 module Protobuf
   module Rpc
     module Zmq
       class Server
         include ::Protobuf::Rpc::Zmq::Util
 
-        attr_accessor :options, :workers, :zmq_context
+        attr_accessor :options
 
         def initialize(options)
           @options = default_options.merge(options)
@@ -138,7 +136,7 @@ def shutdown_uri
         def signal_shutdown
           socket = @zmq_context.socket ZMQ::PAIR
           zmq_error_check(socket.connect shutdown_uri)
-          zmq_error_check(socket.send_string "blargh!")
+          zmq_error_check(socket.send_string ".")
           zmq_error_check(socket.close)
         end
 
@@ -180,7 +178,7 @@ def teardown
         end
 
         def total_workers
-          @total_workers ||= @options[:threads]
+          @total_workers ||= [@options[:threads].to_i, 1].max
         end
 
         def uuid
@@ -230,7 +228,7 @@ def init_beacon_socket
 
         def init_shutdown_socket
           @shutdown_socket = @zmq_context.socket ZMQ::PAIR
-          @shutdown_socket.bind shutdown_uri
+          zmq_error_check(@shutdown_socket.bind shutdown_uri)
         end
 
         def init_zmq_context
diff --git a/lib/protobuf/rpc/servers/zmq/util.rb b/lib/protobuf/rpc/servers/zmq/util.rb
index 513eaa45..6da90ac4 100644
--- a/lib/protobuf/rpc/servers/zmq/util.rb
+++ b/lib/protobuf/rpc/servers/zmq/util.rb
@@ -5,7 +5,6 @@ module Rpc
     module Zmq
 
       WORKER_READY_MESSAGE = "\1"
-      WORKER_SHUTDOWN_MESSAGE = "\2"
 
       module Util
         include ::Protobuf::Logger::LogMethods
diff --git a/lib/protobuf/rpc/servers/zmq/worker.rb b/lib/protobuf/rpc/servers/zmq/worker.rb
index 34c5f52f..89ac5f46 100644
--- a/lib/protobuf/rpc/servers/zmq/worker.rb
+++ b/lib/protobuf/rpc/servers/zmq/worker.rb
@@ -8,8 +8,6 @@ class Worker
         include ::Protobuf::Rpc::Server
         include ::Protobuf::Rpc::Zmq::Util
 
-        attr_reader :thread
-
         ##
         # Constructor
         #
@@ -18,6 +16,9 @@ def initialize(server)
           init_zmq_context
           init_backend_socket
           init_shutdown_socket
+        rescue
+          teardown
+          raise
         end
 
         ##
@@ -27,8 +28,12 @@ def alive?
           @thread.try(:alive?) || false
         end
 
-        def handle_request(socket)
-          zmq_error_check(socket.recv_strings(frames = []))
+        def join
+          @thread.try :join
+        end
+
+        def process_request
+          zmq_error_check(@backend_socket.recv_strings(frames = []))
 
           @client_address, empty, @request_data = *frames
 
@@ -38,31 +43,26 @@ def handle_request(socket)
           end
         end
 
-        def join
-          @thread.try :join
-        end
-
         def run
-          running = true
           poller = ::ZMQ::Poller.new
-
           poller.register_readable(@backend_socket)
           poller.register_readable(@shutdown_socket)
 
           # Send request to broker telling it we are ready
           zmq_error_check(@backend_socket.send_string(::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE))
 
-          while poller.poll > 0
-            poller.readables.each do |readable|
-              if readable === @backend_socket
-                initialize_request!
-                handle_request(@backend_socket)
-              elsif readable === @shutdown_socket
-                running = false
+          catch(:shutdown) do
+            while poller.poll > 0
+              poller.readables.each do |readable|
+                case readable
+                when @backend_socket
+                  initialize_request!
+                  process_request
+                when @shutdown_socket
+                  throw :shutdown
+                end
               end
             end
-
-            break unless running
           end
         ensure
           teardown
@@ -85,7 +85,7 @@ def shutdown_uri
         def signal_shutdown
           socket = @zmq_context.socket ZMQ::PAIR
           zmq_error_check(socket.connect shutdown_uri)
-          zmq_error_check(socket.send_string "shutdown")
+          zmq_error_check(socket.send_string ".")
           zmq_error_check(socket.close)
         end
 
diff --git a/lib/protobuf/rpc/service.rb b/lib/protobuf/rpc/service.rb
index 4246fbd0..aeea555f 100644
--- a/lib/protobuf/rpc/service.rb
+++ b/lib/protobuf/rpc/service.rb
@@ -103,7 +103,7 @@ def self.rpc_method?(name)
 
       # An array of defined service classes
       def self.services
-        @_subclasses.select do |subclass|
+        (@_subclasses ||[]).select do |subclass|
           subclass.rpcs.any? do |(name, method)|
             subclass.method_defined? name
           end

From 8f1300640661fe090668e106ced9be8163da2af5 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 13:56:57 -0600
Subject: [PATCH 25/70] Improve readability

---
 lib/protobuf/rpc/service.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/protobuf/rpc/service.rb b/lib/protobuf/rpc/service.rb
index aeea555f..d3c98625 100644
--- a/lib/protobuf/rpc/service.rb
+++ b/lib/protobuf/rpc/service.rb
@@ -103,7 +103,7 @@ def self.rpc_method?(name)
 
       # An array of defined service classes
       def self.services
-        (@_subclasses ||[]).select do |subclass|
+        (@_subclasses || []).select do |subclass|
           subclass.rpcs.any? do |(name, method)|
             subclass.method_defined? name
           end

From b29765919da2ef1c722099bb68f1759849423d08 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 15:23:46 -0600
Subject: [PATCH 26/70] Rename the beacon command line option

---
 lib/protobuf/cli.rb                    | 2 +-
 lib/protobuf/rpc/servers/zmq/server.rb | 2 +-
 spec/functional/zmq_server_spec.rb     | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index 391be744..c1810fce 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -27,7 +27,7 @@ class CLI < ::Thor
     option :evented,                    :type => :boolean, :aliases => %w(-m), :desc => 'Evented Mode for server and client connections (uses EventMachine).'
     option :zmq,                        :type => :boolean, :aliases => %w(-z), :desc => 'ZeroMQ Socket Mode for server and client connections.'
 
-    option :dynamic_discovery,          :type => :boolean, :default => false, :aliases => %w(-y), :desc => 'Enable dynamic discovery (Currently only available with ZeroMQ).'
+    option :broadcast_beacons,          :type => :boolean, :default => false, :desc => 'Broadcast beacons for dynamic discovery (Currently only available with ZeroMQ).'
     option :debug,                      :type => :boolean, :default => false, :aliases => %w(-d), :desc => 'Debug Mode. Override log level to DEBUG.'
     option :gc_pause_request,           :type => :boolean, :default => false, :desc => 'Enable/Disable GC pause during request.'
     option :print_deprecation_warnings, :type => :boolean, :default => nil, :desc => 'Cause use of deprecated fields to be printed or ignored.'
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 6c773692..c7d9c5e3 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -49,7 +49,7 @@ def beacon_port
         end
 
         def broadcast_beacons?
-          !brokerless? && options[:dynamic_discovery]
+          !brokerless? && options[:broadcast_beacons]
         end
 
         def broadcast_flatline
diff --git a/spec/functional/zmq_server_spec.rb b/spec/functional/zmq_server_spec.rb
index 752fd4b6..506720d8 100644
--- a/spec/functional/zmq_server_spec.rb
+++ b/spec/functional/zmq_server_spec.rb
@@ -9,7 +9,7 @@
     :backlog => 100,
     :threshold => 100,
     :threads => 5,
-    :dynamic_discovery => true,
+    :broadcast_beacons => true,
     :beacon_port => 9398
   }}
 

From 0daf17ff980f6d4dc16a056dbd387f00b7dedb61 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 17:09:18 -0600
Subject: [PATCH 27/70] Use a constant for default options

---
 lib/protobuf/rpc/servers/zmq/server.rb | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index c7d9c5e3..7864a8ad 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -10,10 +10,12 @@ module Zmq
       class Server
         include ::Protobuf::Rpc::Zmq::Util
 
+        DEFAULT_OPTIONS = { :beacon_interval => 5 }
+
         attr_accessor :options
 
         def initialize(options)
-          @options = default_options.merge(options)
+          @options = DEFAULT_OPTIONS.merge(options)
           @workers = []
 
           init_zmq_context
@@ -216,10 +218,6 @@ def wait_for_shutdown_signal
 
         private
 
-        def default_options
-          { :beacon_interval => 5  }
-        end
-
         def init_beacon_socket
           @beacon_socket = UDPSocket.new
           @beacon_socket.setsockopt :SOCKET, :BROADCAST, true

From 46fcea0eb69f7d78bdf67c4f8b8c6dfef6190644 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 29 May 2013 17:09:43 -0600
Subject: [PATCH 28/70] Cleanup whitespace

---
 lib/protobuf.rb | 32 ++++++++++++++++----------------
 1 file changed, 16 insertions(+), 16 deletions(-)

diff --git a/lib/protobuf.rb b/lib/protobuf.rb
index 3c8edc90..24523879 100644
--- a/lib/protobuf.rb
+++ b/lib/protobuf.rb
@@ -15,7 +15,7 @@ module Protobuf
   # Default is Socket as it has no external dependencies.
   DEFAULT_CONNECTOR = :socket
 
-	module_function
+  module_function
 
   # Client Host
   #
@@ -53,14 +53,14 @@ def self.connector_type=(type)
   # the Garbage Collector when handling an rpc request.
   # Once the request is completed, the GC is enabled again.
   # This optomization provides a huge boost in speed to rpc requests.
-	def self.gc_pause_server_request?
-		return @_gc_pause_server_request unless @_gc_pause_server_request.nil?
-		gc_pause_server_request = false
-	end
+  def self.gc_pause_server_request?
+    return @_gc_pause_server_request unless @_gc_pause_server_request.nil?
+    gc_pause_server_request = false
+  end
 
-	def self.gc_pause_server_request=(value)
-		@_gc_pause_server_request = !!value
-	end
+  def self.gc_pause_server_request=(value)
+    @_gc_pause_server_request = !!value
+  end
 
   # Print Deprecation Warnings
   #
@@ -72,14 +72,14 @@ def self.gc_pause_server_request=(value)
   # ENV['PB_IGNORE_DEPRECATIONS'] to a non-empty value.
   #
   # The rpc_server option will override the ENV setting.
-	def self.print_deprecation_warnings?
-		return @_print_deprecation_warnings unless @_print_deprecation_warnings.nil?
-		print_deprecation_warnings = ENV.key?('PB_IGNORE_DEPRECATIONS') ? false : true
-	end
-
-	def self.print_deprecation_warnings=(value)
-		@_print_deprecation_warnings = !!value
-	end
+  def self.print_deprecation_warnings?
+    return @_print_deprecation_warnings unless @_print_deprecation_warnings.nil?
+    print_deprecation_warnings = ENV.key?('PB_IGNORE_DEPRECATIONS') ? false : true
+  end
+
+  def self.print_deprecation_warnings=(value)
+    @_print_deprecation_warnings = !!value
+  end
 
 end
 

From 9f02740d357239f4a1db6af8740c7bb575f59a28 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Thu, 30 May 2013 10:18:48 -0600
Subject: [PATCH 29/70] Don't shutdown the broker pre-maturely

---
 lib/protobuf/rpc/servers/zmq/server.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index c6af7f55..f393cc9c 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -25,7 +25,7 @@ def self.run(options = {})
 
           @running = true
           log_debug { sign_message("server started") }
-          while self.running? do
+          while @threads.any? do
             @broker.poll
           end
         ensure
@@ -53,8 +53,8 @@ def self.start_worker
         def self.stop
           @running = false
 
-          @threads.each do |t|
-            t.join
+          @threads.delete_if do |t|
+            t.join || true
           end
         end
 

From bbcd4b163616e459a8e485efb232d2b38fd6ea94 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 31 May 2013 13:59:33 -0600
Subject: [PATCH 30/70] Add the service directory class

---
 lib/protobuf/rpc/connectors/zmq.rb       |  25 +++-
 lib/protobuf/rpc/dynamic_discovery.pb.rb |  59 +++++----
 lib/protobuf/rpc/servers/zmq/server.rb   |  31 +++--
 lib/protobuf/rpc/service.rb              |   9 +-
 lib/protobuf/rpc/service_directory.rb    | 156 +++++++++++++++++++++++
 proto/dynamic_discovery.proto            |  16 ++-
 6 files changed, 243 insertions(+), 53 deletions(-)
 create mode 100644 lib/protobuf/rpc/service_directory.rb

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index 993ac3e1..bc511b81 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -1,4 +1,5 @@
 require 'protobuf/rpc/connectors/base'
+require 'protobuf/rpc/service_directory'
 
 module Protobuf
   module Rpc
@@ -15,8 +16,8 @@ def send_request
             read_response
           end
         ensure
-          @socket.close if @socket
-          @zmq_context.terminate if @zmq_context
+          @socket.try :close
+          @zmq_context.try :terminate
           @zmq_context = nil
         end
 
@@ -24,6 +25,20 @@ def log_signature
           @_log_signature ||= "[client-#{self.class}]"
         end
 
+        def service_directory
+          ::Protobuf::Rpc::ServiceDirectory.instance
+        end
+
+        def service_uri
+          if service_directory.running?
+            host, port = service_directory.find options[:service]
+          else
+            host, port = options[:host], options[:port]
+          end
+
+          "tcp://#{host}:#{port}"
+        end
+
         private
 
         def close_connection
@@ -35,11 +50,11 @@ def close_connection
 
         def connect_to_rpc_server
           return if @error
-          log_debug { sign_message("Establishing connection: #{options[:host]}:#{options[:port]}") }
+          log_debug { sign_message("Establishing connection: #{service_uri}") }
           @zmq_context = ::ZMQ::Context.new
           @socket = @zmq_context.socket(::ZMQ::REQ)
-          zmq_error_check(@socket.connect("tcp://#{options[:host]}:#{options[:port]}"))
-          log_debug { sign_message("Connection established #{options[:host]}:#{options[:port]}") }
+          zmq_error_check(@socket.connect(service_uri))
+          log_debug { sign_message("Connection established #{service_uri}") }
         end
 
         # Method to determine error state, must be used with Connector api
diff --git a/lib/protobuf/rpc/dynamic_discovery.pb.rb b/lib/protobuf/rpc/dynamic_discovery.pb.rb
index 056ec020..51d0d79e 100644
--- a/lib/protobuf/rpc/dynamic_discovery.pb.rb
+++ b/lib/protobuf/rpc/dynamic_discovery.pb.rb
@@ -5,35 +5,44 @@
 
 module Protobuf
 
-  module DynamicDiscovery
+  module Rpc
   
-    ##
-    # Enum Classes
-    #
-    class BeaconType < ::Protobuf::Enum
-      define :HEARTBEAT, 0
-      define :FLATLINE, 1
-    end
-    
+    module DynamicDiscovery
     
-    ##
-    # Message Classes
-    #
-    class Beacon < ::Protobuf::Message; end
-    
-    ##
-    # Message Fields
-    #
-    class Beacon
-      optional ::Protobuf::DynamicDiscovery::BeaconType, :beacon_type, 1
-      optional ::Protobuf::Field::StringField, :uuid, 2
-      optional ::Protobuf::Field::StringField, :address, 3
-      optional ::Protobuf::Field::StringField, :port, 4
-      optional ::Protobuf::Field::Int32Field, :ttl, 5
-      repeated ::Protobuf::Field::StringField, :service_classes, 6
+      ##
+      # Enum Classes
+      #
+      class BeaconType < ::Protobuf::Enum
+        define :HEARTBEAT, 0
+        define :FLATLINE, 1
+      end
+      
+      
+      ##
+      # Message Classes
+      #
+      class Server < ::Protobuf::Message; end
+      class Beacon < ::Protobuf::Message; end
+      
+      ##
+      # Message Fields
+      #
+      class Server
+        optional ::Protobuf::Field::StringField, :uuid, 1
+        optional ::Protobuf::Field::StringField, :address, 2
+        optional ::Protobuf::Field::StringField, :port, 3
+        optional ::Protobuf::Field::Int32Field, :ttl, 4
+        repeated ::Protobuf::Field::StringField, :services, 5
+      end
+      
+      class Beacon
+        optional ::Protobuf::Rpc::DynamicDiscovery::BeaconType, :beacon_type, 1
+        optional ::Protobuf::Rpc::DynamicDiscovery::Server, :server, 2
+      end
+      
+      
     end
     
-    
   end
   
 end
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 7864a8ad..09e99f70 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -55,28 +55,21 @@ def broadcast_beacons?
         end
 
         def broadcast_flatline
-          flatline = ::Protobuf::DynamicDiscovery::Beacon.new(
-            :beacon_type => ::Protobuf::DynamicDiscovery::BeaconType::FLATLINE,
-            :uuid => uuid,
-            :address => frontend_ip,
-            :port => frontend_port.to_s
+          flatline = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
+            :beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE,
+            :server => self.to_proto
           )
 
           @beacon_socket.send flatline.to_s, 0
         end
 
         def broadcast_heartbeat
-          services = ::Protobuf::Rpc::Service.services
-          heartbeat = ::Protobuf::DynamicDiscovery::Beacon.new(
-            :beacon_type => ::Protobuf::DynamicDiscovery::BeaconType::HEARTBEAT,
-            :uuid => uuid,
-            :address => frontend_ip,
-            :port => frontend_port.to_s,
-            :ttl => beacon_interval * 3,
-            :service_classes => services.map(&:name)
+          heartbeat = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
+            :beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT,
+            :server => self.to_proto
           )
 
-          @beacon_socket.send heartbeat.to_s, 0
+          @beacon_socket.send heartbeat.serialize_to_string, 0
         end
 
         def brokerless?
@@ -183,6 +176,16 @@ def total_workers
           @total_workers ||= [@options[:threads].to_i, 1].max
         end
 
+        def to_proto
+          @proto ||= ::Protobuf::Rpc::DynamicDiscovery::Server.new(
+            :uuid => uuid,
+            :address => frontend_ip,
+            :port => frontend_port.to_s,
+            :ttl => beacon_interval * 3,
+            :services => ::Protobuf::Rpc::Service.implemented_services
+          )
+        end
+
         def uuid
           @uuid ||= SecureRandom.uuid
         end
diff --git a/lib/protobuf/rpc/service.rb b/lib/protobuf/rpc/service.rb
index d3c98625..50b221a6 100644
--- a/lib/protobuf/rpc/service.rb
+++ b/lib/protobuf/rpc/service.rb
@@ -101,13 +101,16 @@ def self.rpc_method?(name)
         rpcs.key?(name)
       end
 
-      # An array of defined service classes
-      def self.services
-        (@_subclasses || []).select do |subclass|
+      # An array of defined service classes that contain implementation
+      # code
+      def self.implemented_services
+        classes = (@_subclasses || []).select do |subclass|
           subclass.rpcs.any? do |(name, method)|
             subclass.method_defined? name
           end
         end
+
+        classes.map &:name
       end
 
       ##
diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
new file mode 100644
index 00000000..cb373490
--- /dev/null
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -0,0 +1,156 @@
+require 'delegate'
+require 'singleton'
+require 'socket'
+require 'thread'
+require 'protobuf/rpc/dynamic_discovery.pb'
+
+module Protobuf
+  module Rpc
+    class ServiceDirectory
+      include ::Singleton
+      include ::Protobuf::Logger::LogMethods
+
+      DEFAULT_ADDRESS = "0.0.0.0"
+      DEFAULT_PORT = 9398
+
+      class Server < Delegator
+        attr_reader :expires_at
+
+        def initialize(server)
+          @server = server
+          @expires_at = Time.now.to_i + ttl
+        end
+
+        def current?
+          !expired?
+        end
+
+        def expired?
+          Time.now.to_i >= @expires_at
+        end
+
+        def __getobj__
+          @server
+        end
+      end
+
+      # Class Methods
+      #
+      def self.address
+        @address ||= DEFAULT_ADDRESS
+      end
+
+      def self.address=(value)
+        @address ||= value
+      end
+
+      def self.port
+        @port ||= DEFAULT_PORT
+      end
+
+      def self.port=(value)
+        @port ||= value.to_i
+      end
+
+      def self.start
+        self.instance.start
+      end
+
+      # Instance Methods
+      #
+      def initialize
+        @servers = {}
+        @mutex = Mutex.new
+      end
+
+      def add_server(server)
+        @mutex.synchronize do
+          @servers[server.uuid] = Server.new(server)
+        end
+      end
+
+      def find(service)
+        log_debug { sign_message("searching for #{service}") }
+        Thread.pass until server = random_server_for(service)
+        log_debug { sign_message("found #{service} at #{server}") }
+
+        [server.address, server.port]
+      end
+
+      def random_server_for(service)
+        @mutex.synchronize do
+          servers = @servers.values.select do |server|
+            server.services.any? do |service_class|
+              server.current? && service_class.to_s == service.to_s
+            end
+          end
+
+          servers.sample
+        end
+      end
+
+      def reap_expired_servers
+        @mutex.synchronize do
+          @servers.delete_if do |uuid, server|
+            server.expired?
+          end
+        end
+      end
+
+      def remove_server(server)
+        @mutex.synchronize do
+          @servers.delete server.uuid
+        end
+      end
+
+      def run
+        loop do
+          data, addr = @socket.recvfrom(1024)
+          beacon = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new
+          beacon.parse_from_string(data) rescue nil
+
+          log_debug { sign_message("received beacon: #{beacon.inspect}") }
+
+          case beacon.beacon_type
+          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
+            add_server(beacon.server)
+          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE
+            remove_server(beacon.server)
+          end
+
+          reap_expired_servers
+        end
+      end
+
+      def running?
+        @thread.try(:alive?)
+      end
+
+      def start
+        unless running?
+          log_debug { sign_message("starting service directory") }
+
+          init_socket
+
+          @thread = Thread.new do
+            begin
+              self.run
+            rescue => e
+              log_debug { sign_message("service directory failed: (#{e.class}) #{e.message}") }
+              raise
+            end
+          end
+        end
+
+        self
+      end
+
+      private
+
+      def init_socket
+        @socket = UDPSocket.new
+        @socket.bind(self.class.address, self.class.port)
+      end
+    end
+  end
+end
diff --git a/proto/dynamic_discovery.proto b/proto/dynamic_discovery.proto
index 3854a105..dcecca5f 100644
--- a/proto/dynamic_discovery.proto
+++ b/proto/dynamic_discovery.proto
@@ -22,19 +22,23 @@
 //
 // Protobufs needed for dynamic discovery zmq server and client.
 
-package protobuf.dynamicDiscovery;
+package protobuf.rpc.dynamicDiscovery;
 
 enum BeaconType {
   HEARTBEAT = 0;
   FLATLINE = 1;
 }
 
+message Server {
+  optional string uuid = 1;
+  optional string address = 2;
+  optional string port = 3;
+  optional int32 ttl = 4;
+  repeated string services = 5;
+}
+
 message Beacon {
   optional BeaconType beacon_type = 1;
-  optional string uuid = 2;
-  optional string address = 3;
-  optional string port = 4;
-  optional int32 ttl = 5;
-  repeated string service_classes = 6;
+  optional Server server = 2;
 }
 

From 115b32804f3bed19dde89631dd42f3c4519138fa Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 31 May 2013 17:05:08 -0600
Subject: [PATCH 31/70] Begin writing the directory service spec

---
 lib/protobuf/rpc/service_directory.rb       | 90 ++++++++++++++-------
 spec/functional/zmq_server_spec.rb          |  5 +-
 spec/lib/protobuf/service_directory_spec.rb | 53 ++++++++++++
 3 files changed, 115 insertions(+), 33 deletions(-)
 create mode 100644 spec/lib/protobuf/service_directory_spec.rb

diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index cb373490..c9c781d4 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -2,6 +2,8 @@
 require 'singleton'
 require 'socket'
 require 'thread'
+require 'timeout'
+
 require 'protobuf/rpc/dynamic_discovery.pb'
 
 module Protobuf
@@ -12,8 +14,11 @@ class ServiceDirectory
 
       DEFAULT_ADDRESS = "0.0.0.0"
       DEFAULT_PORT = 9398
+      DEFAULT_TIMEOUT = 5
+
+      ServiceNotFound = Class.new(RuntimeError)
 
-      class Server < Delegator
+      class Listing < Delegator
         attr_reader :expires_at
 
         def initialize(server)
@@ -36,70 +41,86 @@ def __getobj__
 
       # Class Methods
       #
-      def self.address
-        @address ||= DEFAULT_ADDRESS
+      class << self
+        attr_writer :address, :port, :timeout
       end
 
-      def self.address=(value)
-        @address ||= value
+      def self.address
+        @address ||= DEFAULT_ADDRESS
       end
 
       def self.port
         @port ||= DEFAULT_PORT
       end
 
-      def self.port=(value)
-        @port ||= value.to_i
-      end
-
       def self.start
+        yield(self) if block_given?
         self.instance.start
       end
 
+      def self.stop
+        self.instance.stop
+      end
+
+      def self.timeout
+        @timeout ||= DEFAULT_TIMEOUT
+      end
+
       # Instance Methods
       #
       def initialize
-        @servers = {}
+        @listings = {}
         @mutex = Mutex.new
       end
 
-      def add_server(server)
+      def add_listing_for(server)
         @mutex.synchronize do
-          @servers[server.uuid] = Server.new(server)
+          @listings[server.uuid] = Listing.new(server)
         end
       end
 
       def find(service)
+        listing = nil
         log_debug { sign_message("searching for #{service}") }
-        Thread.pass until server = random_server_for(service)
-        log_debug { sign_message("found #{service} at #{server}") }
 
-        [server.address, server.port]
+        Timeout.timeout(self.class.timeout, ServiceNotFound) do
+          Thread.pass until listing = random_listing_for(service)
+        end
+
+        log_debug { sign_message("found #{service} at #{listing.inspect}") }
+
+        [listing.address, listing.port]
+      end
+
+      def listing_count
+        @mutex.synchronize do
+          @listings.count
+        end
       end
 
-      def random_server_for(service)
+      def random_listing_for(service)
         @mutex.synchronize do
-          servers = @servers.values.select do |server|
-            server.services.any? do |service_class|
-              server.current? && service_class.to_s == service.to_s
+          listings = @listings.values.select do |listing|
+            listing.services.any? do |listed_service|
+              listing.current? && listed_service.to_s == service.to_s
             end
           end
 
-          servers.sample
+          listings.sample
         end
       end
 
-      def reap_expired_servers
+      def remove_expired_listings
         @mutex.synchronize do
-          @servers.delete_if do |uuid, server|
-            server.expired?
+          @listings.delete_if do |uuid, listing|
+            listing.expired?
           end
         end
       end
 
-      def remove_server(server)
+      def remove_listing_for(server)
         @mutex.synchronize do
-          @servers.delete server.uuid
+          @listings.delete(server.uuid)
         end
       end
 
@@ -113,17 +134,17 @@ def run
 
           case beacon.beacon_type
           when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
-            add_server(beacon.server)
+            add_listing_for(beacon.server)
           when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE
-            remove_server(beacon.server)
+            remove_listing_for(beacon.server)
           end
 
-          reap_expired_servers
+          remove_expired_listings
         end
       end
 
       def running?
-        @thread.try(:alive?)
+        !!@thread.try(:alive?)
       end
 
       def start
@@ -136,7 +157,7 @@ def start
             begin
               self.run
             rescue => e
-              log_debug { sign_message("service directory failed: (#{e.class}) #{e.message}") }
+              log_debug { sign_message("error: (#{e.class}) #{e.message}") }
               raise
             end
           end
@@ -145,6 +166,15 @@ def start
         self
       end
 
+      def stop
+        @mutex.synchronize do
+          @thread.try(:kill)
+          @listings = {}
+        end
+
+        @socket.try(:close)
+      end
+
       private
 
       def init_socket
diff --git a/spec/functional/zmq_server_spec.rb b/spec/functional/zmq_server_spec.rb
index 506720d8..28455db3 100644
--- a/spec/functional/zmq_server_spec.rb
+++ b/spec/functional/zmq_server_spec.rb
@@ -1,5 +1,6 @@
 require 'spec_helper'
 require 'spec/support/test/resource_service'
+require 'protobuf/rpc/service_directory'
 
 describe 'Functional ZMQ Client' do
   let(:options) {{
@@ -8,9 +9,7 @@
     :worker_port => 9400,
     :backlog => 100,
     :threshold => 100,
-    :threads => 5,
-    :broadcast_beacons => true,
-    :beacon_port => 9398
+    :threads => 5
   }}
 
   let(:server) { ::Protobuf::Rpc::Zmq::Server.new(options) }
diff --git a/spec/lib/protobuf/service_directory_spec.rb b/spec/lib/protobuf/service_directory_spec.rb
new file mode 100644
index 00000000..e0cae3e9
--- /dev/null
+++ b/spec/lib/protobuf/service_directory_spec.rb
@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+require 'protobuf/rpc/service_directory'
+
+describe ::Protobuf::Rpc::ServiceDirectory do
+  let(:instance) { ::Protobuf::Rpc::ServiceDirectory.instance }
+  let(:server) { double('server', :uuid => '123',
+                                  :services => ['Known::Service'],
+                                  :address => "0.0.0.0",
+                                  :port => 9999,
+                                  :ttl => 15) }
+
+  before(:all) do
+    ::Protobuf::Rpc::ServiceDirectory.timeout = 0.5
+  end
+
+  after do
+    instance.stop
+  end
+
+  it "is a singleton" do
+    instance.should be_a_kind_of(Singleton)
+  end
+
+  describe "#find" do
+    it "times out when nothing is found" do
+      expect {
+        instance.find("Unknown::Service")
+      }.to raise_error(::Protobuf::Rpc::ServiceDirectory::ServiceNotFound)
+    end
+
+    it "returns a listing for the given service" do
+      instance.add_listing_for(server)
+      instance.find("Known::Service").should eq [server.address, server.port]
+    end
+  end
+
+  describe "#remove_expired_listings" do
+    before do
+      instance.instance_variable_set(:@listings, {
+        '1' => double(:expired? => true),
+        '2' => double(:expired? => true),
+        '3' => double(:expired? => false),
+      })
+    end
+
+    it "removes expired listings" do
+      expect {
+        instance.remove_expired_listings
+      }.to change(instance, :listing_count).from(3).to(1)
+    end
+  end
+end

From 0f4b3c85d7d357a683b746260755ec65408a4dce Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Mon, 3 Jun 2013 10:54:03 -0600
Subject: [PATCH 32/70] Move spec to the correct folder

---
 spec/lib/protobuf/{ => rpc}/service_directory_spec.rb | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename spec/lib/protobuf/{ => rpc}/service_directory_spec.rb (100%)

diff --git a/spec/lib/protobuf/service_directory_spec.rb b/spec/lib/protobuf/rpc/service_directory_spec.rb
similarity index 100%
rename from spec/lib/protobuf/service_directory_spec.rb
rename to spec/lib/protobuf/rpc/service_directory_spec.rb

From fccf0150b0602204701b3e911b05aa9077a8f0e9 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Mon, 3 Jun 2013 15:27:53 -0600
Subject: [PATCH 33/70] Test and improve directory service

---
 lib/protobuf/rpc/connectors/zmq.rb            |   6 +-
 lib/protobuf/rpc/servers/zmq/server.rb        |   2 +-
 lib/protobuf/rpc/service_directory.rb         |  83 +++++++------
 .../protobuf/rpc/service_directory_spec.rb    | 116 ++++++++++++++++--
 4 files changed, 159 insertions(+), 48 deletions(-)

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index bc511b81..0180dee9 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -31,12 +31,12 @@ def service_directory
 
         def service_uri
           if service_directory.running?
-            host, port = service_directory.find options[:service]
+            address, port = service_directory.find options[:service]
           else
-            host, port = options[:host], options[:port]
+            address, port = options[:host], options[:port]
           end
 
-          "tcp://#{host}:#{port}"
+          "tcp://#{address}:#{port}"
         end
 
         private
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 09e99f70..fff03713 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -60,7 +60,7 @@ def broadcast_flatline
             :server => self.to_proto
           )
 
-          @beacon_socket.send flatline.to_s, 0
+          @beacon_socket.send flatline.serialize_to_string, 0
         end
 
         def broadcast_heartbeat
diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index c9c781d4..f51c1a8c 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -26,6 +26,10 @@ def initialize(server)
           @expires_at = Time.now.to_i + ttl
         end
 
+        def <=>(other)
+          other.expires_at <=> self.expires_at
+        end
+
         def current?
           !expired?
         end
@@ -74,8 +78,15 @@ def initialize
       end
 
       def add_listing_for(server)
-        @mutex.synchronize do
-          @listings[server.uuid] = Listing.new(server)
+        if server && server.uuid
+          log_debug { sign_message("Adding listing for server: #{server.inspect}") }
+
+          @mutex.synchronize do
+            @listings[server.uuid] = Listing.new(server)
+          end
+
+        else
+          log_debug { sign_message("Cannot add server without uuid: #{server.inspect}") }
         end
       end
 
@@ -84,7 +95,7 @@ def find(service)
         log_debug { sign_message("searching for #{service}") }
 
         Timeout.timeout(self.class.timeout, ServiceNotFound) do
-          Thread.pass until listing = random_listing_for(service)
+          Thread.pass until listing = youngest_listing_for(service)
         end
 
         log_debug { sign_message("found #{service} at #{listing.inspect}") }
@@ -92,24 +103,6 @@ def find(service)
         [listing.address, listing.port]
       end
 
-      def listing_count
-        @mutex.synchronize do
-          @listings.count
-        end
-      end
-
-      def random_listing_for(service)
-        @mutex.synchronize do
-          listings = @listings.values.select do |listing|
-            listing.services.any? do |listed_service|
-              listing.current? && listed_service.to_s == service.to_s
-            end
-          end
-
-          listings.sample
-        end
-      end
-
       def remove_expired_listings
         @mutex.synchronize do
           @listings.delete_if do |uuid, listing|
@@ -119,18 +112,25 @@ def remove_expired_listings
       end
 
       def remove_listing_for(server)
-        @mutex.synchronize do
-          @listings.delete(server.uuid)
+        if server && server.uuid
+          log_debug { sign_message("Removing server: #{server.inspect}") }
+
+          @mutex.synchronize do
+            @listings.delete(server.uuid)
+          end
+
+        else
+          log_debug { sign_message("Cannot remove server without uuid: #{server.inspect}") }
         end
       end
 
       def run
-        loop do
+        begin
           data, addr = @socket.recvfrom(1024)
           beacon = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new
           beacon.parse_from_string(data) rescue nil
 
-          log_debug { sign_message("received beacon: #{beacon.inspect}") }
+          log_debug { sign_message("received beacon: #{beacon.inspect} from #{addr}") }
 
           case beacon.beacon_type
           when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
@@ -140,7 +140,9 @@ def run
           end
 
           remove_expired_listings
-        end
+        rescue => e
+          log_debug { sign_message("error: (#{e.class}) #{e.message}") }
+        end while true
       end
 
       def running?
@@ -149,30 +151,37 @@ def running?
 
       def start
         unless running?
-          log_debug { sign_message("starting service directory") }
-
+          log_debug { sign_message("starting") }
           init_socket
-
-          @thread = Thread.new do
-            begin
-              self.run
-            rescue => e
-              log_debug { sign_message("error: (#{e.class}) #{e.message}") }
-              raise
-            end
-          end
+          @thread = Thread.new { self.run }
         end
 
         self
       end
 
       def stop
+        log_debug { sign_message("stopping") }
+
         @mutex.synchronize do
           @thread.try(:kill)
+          @thread = nil
           @listings = {}
         end
 
         @socket.try(:close)
+        @socket = nil
+      end
+
+      def youngest_listing_for(service)
+        @mutex.synchronize do
+          listings = @listings.values.select do |listing|
+            listing.services.any? do |listed_service|
+              listing.current? && listed_service.to_s == service.to_s
+            end
+          end
+
+          listings.sort.first
+        end
       end
 
       private
diff --git a/spec/lib/protobuf/rpc/service_directory_spec.rb b/spec/lib/protobuf/rpc/service_directory_spec.rb
index e0cae3e9..6a31dfc2 100644
--- a/spec/lib/protobuf/rpc/service_directory_spec.rb
+++ b/spec/lib/protobuf/rpc/service_directory_spec.rb
@@ -4,14 +4,13 @@
 
 describe ::Protobuf::Rpc::ServiceDirectory do
   let(:instance) { ::Protobuf::Rpc::ServiceDirectory.instance }
-  let(:server) { double('server', :uuid => '123',
-                                  :services => ['Known::Service'],
-                                  :address => "0.0.0.0",
-                                  :port => 9999,
-                                  :ttl => 15) }
+
+  def listings
+    instance.instance_variable_get(:@listings)
+  end
 
   before(:all) do
-    ::Protobuf::Rpc::ServiceDirectory.timeout = 0.5
+    ::Protobuf::Rpc::ServiceDirectory.timeout = 0.1
   end
 
   after do
@@ -23,6 +22,12 @@
   end
 
   describe "#find" do
+    let(:server) { double('server', :uuid => '123',
+                          :services => ['Known::Service'],
+                          :address => "0.0.0.0",
+                          :port => 9999,
+                          :ttl => 15) }
+
     it "times out when nothing is found" do
       expect {
         instance.find("Unknown::Service")
@@ -33,6 +38,20 @@
       instance.add_listing_for(server)
       instance.find("Known::Service").should eq [server.address, server.port]
     end
+
+    it "does not return expired listings" do
+      server.stub(:ttl => 0)
+      instance.add_listing_for(server)
+      expect {
+        instance.find("Known::Service")
+      }.to raise_error(::Protobuf::Rpc::ServiceDirectory::ServiceNotFound)
+    end
+
+    it "delegates to #youngest_listing_for" do
+      instance.should_receive(:youngest_listing_for).with("Known::Service") { double(:address => '0.0.0.0', :port => 9398) }
+      instance.add_listing_for(server)
+      instance.find("Known::Service")
+    end
   end
 
   describe "#remove_expired_listings" do
@@ -47,7 +66,90 @@
     it "removes expired listings" do
       expect {
         instance.remove_expired_listings
-      }.to change(instance, :listing_count).from(3).to(1)
+      }.to change(listings, :size).from(3).to(1)
+      listings.keys.should eq ['3']
+    end
+  end
+
+  describe "#start" do
+    it "creates a thread" do
+      Thread.should_receive(:new)
+      instance.start
+    end
+
+    it "initializes the socket" do
+      instance.should_receive :init_socket
+      instance.start
+    end
+
+    it "calls #run" do
+      instance.should_receive(:run)
+      instance.start
+      sleep 0.01
+    end
+
+    it "changes the running state" do
+      expect {
+        instance.start
+      }.to change(instance, :running?).from(false).to(true)
+    end
+  end
+
+  describe "#youngest_listing_for" do
+    it "returns the youngest listing" do
+      instance.add_listing_for double(:uuid => 1, :ttl => 5, :services => ["Test"])
+      instance.add_listing_for double(:uuid => 2, :ttl => 15, :services => ["Test"])
+      instance.add_listing_for double(:uuid => 3, :ttl => 10, :services => ["Test"])
+      instance.youngest_listing_for("Test").uuid.should eq 2
+    end
+  end
+
+  describe "a running service directory" do
+    let(:socket) { UDPSocket.new }
+
+    def thread
+      instance.instance_variable_get(:@thread)
+    end
+
+    before do
+      described_class.start do |config|
+        config.address = "127.0.0.1"
+        config.port = 33333
+      end
+
+      socket.connect described_class.address, described_class.port
+    end
+
+    context "receiving a heartbeat" do
+      let(:server) { ::Protobuf::Rpc::DynamicDiscovery::Server.new(:uuid => 'heartbeat') }
+      let(:beacon) { ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
+        :server => server,
+        :beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
+      )}
+      let(:payload) { beacon.serialize_to_string }
+
+      it "adds a listing" do
+        instance.should_receive(:add_listing_for).with(server)
+        instance.should_receive(:remove_expired_listings)
+        socket.send(payload, 0)
+        sleep 0.01
+      end
+    end
+
+    context "receiving a flatline" do
+      let(:server) { ::Protobuf::Rpc::DynamicDiscovery::Server.new(:uuid => 'flatline') }
+      let(:beacon) { ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
+        :server => server,
+        :beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE
+      )}
+      let(:payload) { beacon.serialize_to_string }
+
+      it "removes a listing" do
+        instance.should_receive(:remove_listing_for).with(server)
+        instance.should_receive(:remove_expired_listings)
+        socket.send(payload, 0)
+        sleep 0.01
+      end
     end
   end
 end

From 69fb457c3edbea16166a2efccac116bda5ba7d7d Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Mon, 3 Jun 2013 16:08:36 -0600
Subject: [PATCH 34/70] Incorporate the lazy pirate from @localshred

---
 lib/protobuf/rpc/connectors/zmq.rb | 202 ++++++++++++++++++++++-------
 1 file changed, 158 insertions(+), 44 deletions(-)

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index 0180dee9..b55403a6 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -5,81 +5,195 @@ module Protobuf
   module Rpc
     module Connectors
       class Zmq < Base
+
+        ##
+        # Included Modules
+        #
+
         include Protobuf::Rpc::Connectors::Common
         include Protobuf::Logger::LogMethods
 
+        ##
+        # Class Constants
+        #
+
+        CLIENT_RETRIES = (ENV['PB_CLIENT_RETRIES'] || 3)
+
+        ##
+        # Instance methods
+        #
+
+        # Start the request/response cycle. We implement the Lazy Pirate
+        # req/reply reliability pattern as laid out in the ZMQ Guide, Chapter 4.
+        #
+        # @see http://zguide.zeromq.org/php:chapter4#Client-side-Reliability-Lazy-Pirate-Pattern
+        #
         def send_request
-          timeout_wrap do
-            setup_connection
-            connect_to_rpc_server
-            post_init
-            read_response
-          end
+          setup_connection
+          poll_send_data
         ensure
-          @socket.try :close
-          @zmq_context.try :terminate
-          @zmq_context = nil
+          close_connection
         end
 
         def log_signature
           @_log_signature ||= "[client-#{self.class}]"
         end
 
-        def service_directory
-          ::Protobuf::Rpc::ServiceDirectory.instance
-        end
-
-        def service_uri
-          if service_directory.running?
-            address, port = service_directory.find options[:service]
-          else
-            address, port = options[:host], options[:port]
-          end
-
-          "tcp://#{address}:#{port}"
-        end
-
         private
 
+        ##
+        # Private Instance methods
+        #
+
         def close_connection
-          return if @error
-          zmq_error_check(@socket.close)
-          zmq_error_check(@zmq_context.terminate)
-          log_debug { sign_message("Connector closed")  }
+          socket_close
+          zmq_context_terminate
         end
 
+        # Establish a request socket connection to the remote rpc_server.
+        # Set the socket option LINGER to 0 so that we don't wait
+        # for queued messages to be accepted when the socket/context are
+        # asked to close/terminate.
+        #
         def connect_to_rpc_server
-          return if @error
-          log_debug { sign_message("Establishing connection: #{service_uri}") }
-          @zmq_context = ::ZMQ::Context.new
-          @socket = @zmq_context.socket(::ZMQ::REQ)
-          zmq_error_check(@socket.connect(service_uri))
-          log_debug { sign_message("Connection established #{service_uri}") }
+          return if error?
+
+          log_debug { sign_message("Establishing connection: #{server_uri}") }
+          socket.setsockopt(::ZMQ::LINGER, 0)
+          zmq_error_check(socket.connect(server_uri), :socket_connect)
+          zmq_error_check(poller.register_readable(socket), :poller_register_readable)
+          log_debug { sign_message("Connection established to #{server_uri}") }
         end
 
-        # Method to determine error state, must be used with Connector api
+        # Method to determine error state, must be used with Connector API.
+        #
         def error?
-          !!@error
+          !! @error
+        end
+
+        # Trying a number of times, attempt to get a response from the server.
+        # If we haven't received a legitimate response in the CLIENT_RETRIES number
+        # of retries, fail the request.
+        #
+        def poll_send_data
+          return if error?
+
+          poll_timeout = (options[:timeout].to_f / CLIENT_RETRIES.to_f) * 1000
+
+          CLIENT_RETRIES.times do |n|
+            connect_to_rpc_server
+            log_debug { sign_message("Sending Request (attempt #{n + 1}, #{socket})") }
+            send_data
+            log_debug { sign_message("Request sending complete (attempt #{n + 1}, #{socket})") }
+
+            if poller.poll(poll_timeout) == 1
+              read_response
+              return
+            else
+              close_connection
+            end
+          end
+
+          fail(:RPC_FAILED, "The server took longer than #{options[:timeout]} seconds to respond")
+        end
+
+        def poller
+          @poller ||= ::ZMQ::Poller.new
         end
 
+        # Read the string response from the available readable. This will be
+        # the current @socket. Calls `parse_response` to invoke the success or
+        # failed callbacks, depending on the state of the communication
+        # and response data.
+        #
         def read_response
-          return if @error
-          @response_data = ''
-          zmq_error_check(@socket.recv_string(@response_data))
+          return if error?
+
+          poller.readables.each do |readable|
+            @response_data = ''
+            zmq_error_check(readable.recv_string(@response_data), :socket_recv_string)
+          end
+
           parse_response
         end
 
+        # Send the request data to the remote rpc_server.
+        #
         def send_data
-          return if @error
-          log_debug { sign_message("Sending Request: #{@request_data}") }
+          return if error?
+
           @stats.request_size = @request_data.size
-          zmq_error_check(@socket.send_string(@request_data))
-          log_debug { sign_message("write closed") }
+          zmq_error_check(socket.send_string(@request_data), :socket_send_string)
         end
 
-        def zmq_error_check(return_code)
-          raise "Last API call failed at #{caller(1)}" unless return_code >= 0
+        # The location of the server. If the ServiceDirectory is running then
+        # we ask it for a server that hosts the requested service. If it is
+        # not running, or it cannot be found, we fallback to the options.
+        #
+        def server_uri
+          if service_directory.running?
+            listing = service_directory.find(service) rescue nil
+            host, port = listing.address, listing.port if listing
+          end
+
+          host, port = options[:host], options[:port] unless host && port
+
+          "tcp://#{host}:#{port}"
         end
+
+        # The service we're attempting to connect to
+        #
+        def service
+          options[:service]
+        end
+
+        # Alias for ::Protobuf::Rpc::ServiceDirectory.instance
+        def service_directory
+          ::Protobuf::Rpc::ServiceDirectory.instance
+        end
+
+        # Setup a ZMQ request socket in the current zmq context.
+        #
+        def socket
+          @socket ||= zmq_context.socket(::ZMQ::REQ)
+        end
+
+        def socket_close
+          if socket
+            log_debug { sign_message("Closing Socket")  }
+            zmq_error_check(socket.close, :socket_close)
+            log_debug { sign_message("Socket closed")  }
+            @socket = nil
+          end
+        end
+
+        # Return the ZMQ Context to use for this process.
+        # If the context does not exist, create it, then register
+        # an exit block to ensure the context is terminated correctly.
+        #
+        def zmq_context
+          @zmq_context ||= ::ZMQ::Context.new
+        end
+
+        # Terminate the zmq_context (if any).
+        #
+        def zmq_context_terminate
+          log_debug { sign_message("Terminating ZMQ Context")  }
+          @zmq_context.try(:terminate)
+          @zmq_context = nil
+          log_debug { sign_message("ZMQ Context terminated")  }
+        end
+
+        def zmq_error_check(return_code, source)
+          unless ::ZMQ::Util.resultcode_ok?(return_code || -1)
+            raise <<-ERROR
+            Last ZMQ API call to #{source} failed with "#{::ZMQ::Util.error_string}".
+
+            #{caller(1).join($/)}
+            ERROR
+          end
+        end
+
       end
     end
   end

From a489cbc98a2320b19a4bd76b658c57b663ac5791 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Tue, 4 Jun 2013 10:26:02 -0600
Subject: [PATCH 35/70] Add tests around the service directory

---
 lib/protobuf/rpc/servers/socket/server.rb     |  3 +-
 lib/protobuf/rpc/service_directory.rb         |  6 ++-
 spec/lib/protobuf/rpc/connectors/zmq_spec.rb  | 44 ++++++++++++++++++-
 .../protobuf/rpc/service_directory_spec.rb    | 23 +++++-----
 4 files changed, 62 insertions(+), 14 deletions(-)

diff --git a/lib/protobuf/rpc/servers/socket/server.rb b/lib/protobuf/rpc/servers/socket/server.rb
index b837a815..94a38dfa 100644
--- a/lib/protobuf/rpc/servers/socket/server.rb
+++ b/lib/protobuf/rpc/servers/socket/server.rb
@@ -66,8 +66,9 @@ def run
 
           while running?
             log_debug { sign_message("Waiting for connections") }
+            ready_cnxns = IO.select(@listen_fds, [], [], AUTO_COLLECT_TIMEOUT) rescue nil
 
-            if ready_cnxns = IO.select(@listen_fds, [], [], AUTO_COLLECT_TIMEOUT)
+            if ready_cnxns
               cnxns = ready_cnxns.first
               cnxns.each do |client|
                 case
diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index f51c1a8c..09019c90 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -38,6 +38,10 @@ def expired?
           Time.now.to_i >= @expires_at
         end
 
+        def ttl
+          [super.to_i, 3].max
+        end
+
         def __getobj__
           @server
         end
@@ -100,7 +104,7 @@ def find(service)
 
         log_debug { sign_message("found #{service} at #{listing.inspect}") }
 
-        [listing.address, listing.port]
+        listing
       end
 
       def remove_expired_listings
diff --git a/spec/lib/protobuf/rpc/connectors/zmq_spec.rb b/spec/lib/protobuf/rpc/connectors/zmq_spec.rb
index eeca9fc8..8151f752 100644
--- a/spec/lib/protobuf/rpc/connectors/zmq_spec.rb
+++ b/spec/lib/protobuf/rpc/connectors/zmq_spec.rb
@@ -2,6 +2,16 @@
 require 'protobuf/zmq'
 
 describe ::Protobuf::Rpc::Connectors::Zmq do
+  subject { described_class.new(options) }
+
+  let(:options) {{
+    :service => "Test::Service",
+    :method => "find",
+    :timeout => 3,
+    :host => "127.0.0.1",
+    :port => "9400"
+  }}
+
   let(:socket_mock) do
     sm = mock(::ZMQ::Socket)
     sm.stub(:connect).and_return(0)
@@ -14,9 +24,41 @@
     zc
   end
 
-  before(:each) do
+  before do
     ::ZMQ::Context.stub(:new).and_return(zmq_context_mock)
   end
 
+  describe "#server_uri" do
+    let(:service_directory) { double('ServiceDirectory', :running? => running? ) }
+    let(:listing) { double('Listing', :address => '127.0.0.2', :port => 9399) }
+    let(:running?) { true }
+
+    before do
+      subject.stub(:service_directory) { service_directory }
+    end
+
+    context "when the service directory is running" do
+      it "searches the service directory" do
+        service_directory.should_receive(:find).and_return(listing)
+        subject.send(:server_uri).should eq "tcp://127.0.0.2:9399"
+      end
+
+      it "defaults to the options" do
+        service_directory.should_receive(:find).and_raise(RuntimeError)
+        subject.send(:server_uri).should eq "tcp://127.0.0.1:9400"
+      end
+    end
+
+    context "when the service directory is not running" do
+      let(:running?) { false }
+
+      it "does not search the directory" do
+        service_directory.should_not_receive(:find)
+        subject.send(:server_uri).should eq "tcp://127.0.0.1:9400"
+      end
+    end
+
+  end
+
   pending
 end
diff --git a/spec/lib/protobuf/rpc/service_directory_spec.rb b/spec/lib/protobuf/rpc/service_directory_spec.rb
index 6a31dfc2..c7e597bd 100644
--- a/spec/lib/protobuf/rpc/service_directory_spec.rb
+++ b/spec/lib/protobuf/rpc/service_directory_spec.rb
@@ -27,6 +27,7 @@ def listings
                           :address => "0.0.0.0",
                           :port => 9999,
                           :ttl => 15) }
+    let(:listing) { ::Protobuf::Rpc::ServiceDirectory::Listing.new(server) }
 
     it "times out when nothing is found" do
       expect {
@@ -36,20 +37,12 @@ def listings
 
     it "returns a listing for the given service" do
       instance.add_listing_for(server)
-      instance.find("Known::Service").should eq [server.address, server.port]
-    end
-
-    it "does not return expired listings" do
-      server.stub(:ttl => 0)
-      instance.add_listing_for(server)
-      expect {
-        instance.find("Known::Service")
-      }.to raise_error(::Protobuf::Rpc::ServiceDirectory::ServiceNotFound)
+      instance.find("Known::Service").should eq listing
     end
 
     it "delegates to #youngest_listing_for" do
-      instance.should_receive(:youngest_listing_for).with("Known::Service") { double(:address => '0.0.0.0', :port => 9398) }
-      instance.add_listing_for(server)
+      instance.should_receive(:youngest_listing_for)
+              .with("Known::Service") { double('listing') }
       instance.find("Known::Service")
     end
   end
@@ -102,6 +95,14 @@ def listings
       instance.add_listing_for double(:uuid => 3, :ttl => 10, :services => ["Test"])
       instance.youngest_listing_for("Test").uuid.should eq 2
     end
+
+    it "does not return expired listings" do
+      instance.instance_variable_set(:@listings, {
+        '1' => double(:current? => false, :services => ["Test"]),
+      })
+
+      instance.youngest_listing_for("Test").should be_nil
+    end
   end
 
   describe "a running service directory" do

From 93831dc99b81ef07f4b4164b10c60a6e2e3456fd Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Tue, 4 Jun 2013 12:34:54 -0600
Subject: [PATCH 36/70] Add additional options for the zmq server

---
 lib/protobuf/rpc/servers/zmq/server.rb | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index fff03713..90f0b7c2 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -10,7 +10,12 @@ module Zmq
       class Server
         include ::Protobuf::Rpc::Zmq::Util
 
-        DEFAULT_OPTIONS = { :beacon_interval => 5 }
+        DEFAULT_OPTIONS = {
+          :beacon_address => "255.255.255.255",
+          :beacon_interval => 5,
+          :beacon_port => 9398,
+          :broadcast_beacons => false
+        }
 
         attr_accessor :options
 
@@ -43,7 +48,7 @@ def beacon_interval
         end
 
         def beacon_ip
-          "255.255.255.255"
+          @beacon_ip ||= resolve_ip(options[:beacon_address])
         end
 
         def beacon_port

From eab132d0e4ceef78a9b02c57fcfc81a76c1c8618 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Tue, 4 Jun 2013 16:07:55 -0600
Subject: [PATCH 37/70] Refactor DirectoryService#find

---
 lib/protobuf/rpc/connectors/zmq.rb            |  2 +-
 lib/protobuf/rpc/service_directory.rb         | 93 ++++++++-----------
 spec/lib/protobuf/rpc/connectors/zmq_spec.rb  |  6 +-
 .../protobuf/rpc/service_directory_spec.rb    | 77 ++++++++++-----
 4 files changed, 97 insertions(+), 81 deletions(-)

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index b55403a6..863f8bb5 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -132,7 +132,7 @@ def send_data
         #
         def server_uri
           if service_directory.running?
-            listing = service_directory.find(service) rescue nil
+            listing = service_directory.lookup(service)
             host, port = listing.address, listing.port if listing
           end
 
diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index 09019c90..66383827 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -14,9 +14,7 @@ class ServiceDirectory
 
       DEFAULT_ADDRESS = "0.0.0.0"
       DEFAULT_PORT = 9398
-      DEFAULT_TIMEOUT = 5
-
-      ServiceNotFound = Class.new(RuntimeError)
+      DEFAULT_TIMEOUT = 1
 
       class Listing < Delegator
         attr_reader :expires_at
@@ -26,10 +24,6 @@ def initialize(server)
           @expires_at = Time.now.to_i + ttl
         end
 
-        def <=>(other)
-          other.expires_at <=> self.expires_at
-        end
-
         def current?
           !expired?
         end
@@ -50,7 +44,7 @@ def __getobj__
       # Class Methods
       #
       class << self
-        attr_writer :address, :port, :timeout
+        attr_writer :address, :port
       end
 
       def self.address
@@ -70,10 +64,6 @@ def self.stop
         self.instance.stop
       end
 
-      def self.timeout
-        @timeout ||= DEFAULT_TIMEOUT
-      end
-
       # Instance Methods
       #
       def initialize
@@ -94,17 +84,18 @@ def add_listing_for(server)
         end
       end
 
-      def find(service)
-        listing = nil
-        log_debug { sign_message("searching for #{service}") }
-
-        Timeout.timeout(self.class.timeout, ServiceNotFound) do
-          Thread.pass until listing = youngest_listing_for(service)
-        end
+      def lookup(service)
+        log_debug { sign_message("lookup #{service}") }
 
-        log_debug { sign_message("found #{service} at #{listing.inspect}") }
+        @mutex.synchronize do
+          listings = @listings.values.select do |listing|
+            listing.services.any? do |listed_service|
+              listing.current? && listed_service == service.to_s
+            end
+          end
 
-        listing
+          listings.sample
+        end
       end
 
       def remove_expired_listings
@@ -128,27 +119,6 @@ def remove_listing_for(server)
         end
       end
 
-      def run
-        begin
-          data, addr = @socket.recvfrom(1024)
-          beacon = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new
-          beacon.parse_from_string(data) rescue nil
-
-          log_debug { sign_message("received beacon: #{beacon.inspect} from #{addr}") }
-
-          case beacon.beacon_type
-          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
-            add_listing_for(beacon.server)
-          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE
-            remove_listing_for(beacon.server)
-          end
-
-          remove_expired_listings
-        rescue => e
-          log_debug { sign_message("error: (#{e.class}) #{e.message}") }
-        end while true
-      end
-
       def running?
         !!@thread.try(:alive?)
       end
@@ -157,7 +127,7 @@ def start
         unless running?
           log_debug { sign_message("starting") }
           init_socket
-          @thread = Thread.new { self.run }
+          @thread = Thread.new { self.send(:run) }
         end
 
         self
@@ -176,16 +146,14 @@ def stop
         @socket = nil
       end
 
-      def youngest_listing_for(service)
-        @mutex.synchronize do
-          listings = @listings.values.select do |listing|
-            listing.services.any? do |listed_service|
-              listing.current? && listed_service.to_s == service.to_s
-            end
-          end
-
-          listings.sort.first
+      def wait_for(service, timeout = DEFAULT_TIMEOUT)
+        log_debug { sign_message("waiting for #{service}") }
+        Timeout.timeout(timeout) do
+          sleep(timeout / 10.0) until listing = lookup(service)
+          listing
         end
+      rescue
+        nil
       end
 
       private
@@ -194,6 +162,27 @@ def init_socket
         @socket = UDPSocket.new
         @socket.bind(self.class.address, self.class.port)
       end
+
+      def run
+        begin
+          data, addr = @socket.recvfrom(2048)
+          beacon = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new
+          beacon.parse_from_string(data) rescue nil
+
+          log_debug { sign_message("received beacon: #{beacon.inspect} from #{addr}") }
+
+          case beacon.beacon_type
+          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
+            add_listing_for(beacon.server)
+          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE
+            remove_listing_for(beacon.server)
+          end
+
+          remove_expired_listings
+        rescue => e
+          log_debug { sign_message("error: (#{e.class}) #{e.message}") }
+        end while true
+      end
     end
   end
 end
diff --git a/spec/lib/protobuf/rpc/connectors/zmq_spec.rb b/spec/lib/protobuf/rpc/connectors/zmq_spec.rb
index 8151f752..7e32c41e 100644
--- a/spec/lib/protobuf/rpc/connectors/zmq_spec.rb
+++ b/spec/lib/protobuf/rpc/connectors/zmq_spec.rb
@@ -39,12 +39,12 @@
 
     context "when the service directory is running" do
       it "searches the service directory" do
-        service_directory.should_receive(:find).and_return(listing)
+        service_directory.should_receive(:lookup).and_return(listing)
         subject.send(:server_uri).should eq "tcp://127.0.0.2:9399"
       end
 
       it "defaults to the options" do
-        service_directory.should_receive(:find).and_raise(RuntimeError)
+        service_directory.should_receive(:lookup) { nil }
         subject.send(:server_uri).should eq "tcp://127.0.0.1:9400"
       end
     end
@@ -53,7 +53,7 @@
       let(:running?) { false }
 
       it "does not search the directory" do
-        service_directory.should_not_receive(:find)
+        service_directory.should_not_receive(:lookup)
         subject.send(:server_uri).should eq "tcp://127.0.0.1:9400"
       end
     end
diff --git a/spec/lib/protobuf/rpc/service_directory_spec.rb b/spec/lib/protobuf/rpc/service_directory_spec.rb
index c7e597bd..31d41e49 100644
--- a/spec/lib/protobuf/rpc/service_directory_spec.rb
+++ b/spec/lib/protobuf/rpc/service_directory_spec.rb
@@ -9,8 +9,10 @@ def listings
     instance.instance_variable_get(:@listings)
   end
 
-  before(:all) do
-    ::Protobuf::Rpc::ServiceDirectory.timeout = 0.1
+  def duration
+    start = Time.now.to_f
+    yield
+    Time.now.to_f - start
   end
 
   after do
@@ -21,7 +23,7 @@ def listings
     instance.should be_a_kind_of(Singleton)
   end
 
-  describe "#find" do
+  describe "#lookup" do
     let(:server) { double('server', :uuid => '123',
                           :services => ['Known::Service'],
                           :address => "0.0.0.0",
@@ -29,21 +31,26 @@ def listings
                           :ttl => 15) }
     let(:listing) { ::Protobuf::Rpc::ServiceDirectory::Listing.new(server) }
 
-    it "times out when nothing is found" do
-      expect {
-        instance.find("Unknown::Service")
-      }.to raise_error(::Protobuf::Rpc::ServiceDirectory::ServiceNotFound)
-    end
-
     it "returns a listing for the given service" do
       instance.add_listing_for(server)
-      instance.find("Known::Service").should eq listing
+      instance.lookup("Known::Service").should eq listing
+    end
+
+    it "returns random listings" do
+      instance.add_listing_for double(:uuid => 1, :ttl => 5, :services => ["Test"])
+      instance.add_listing_for double(:uuid => 2, :ttl => 5, :services => ["Test"])
+
+      uuids = 100.times.map { instance.lookup("Test").uuid }
+      uuids.count(1).should be_within(25).of(50)
+      uuids.count(2).should be_within(25).of(50)
     end
 
-    it "delegates to #youngest_listing_for" do
-      instance.should_receive(:youngest_listing_for)
-              .with("Known::Service") { double('listing') }
-      instance.find("Known::Service")
+    it "does not return expired listings" do
+      instance.instance_variable_set(:@listings, {
+        '1' => double(:current? => false, :services => ["Test"]),
+      })
+
+      instance.lookup("Test").should be_nil
     end
   end
 
@@ -88,20 +95,40 @@ def listings
     end
   end
 
-  describe "#youngest_listing_for" do
-    it "returns the youngest listing" do
-      instance.add_listing_for double(:uuid => 1, :ttl => 5, :services => ["Test"])
-      instance.add_listing_for double(:uuid => 2, :ttl => 15, :services => ["Test"])
-      instance.add_listing_for double(:uuid => 3, :ttl => 10, :services => ["Test"])
-      instance.youngest_listing_for("Test").uuid.should eq 2
+  describe "#wait_for" do
+    it "returns a listing for the given service" do
+      server = double(:uuid => 1, :ttl => 5, :services => ["Test"])
+      instance.add_listing_for server
+      instance.lookup("Test").should eq server
     end
 
-    it "does not return expired listings" do
-      instance.instance_variable_set(:@listings, {
-        '1' => double(:current? => false, :services => ["Test"]),
-      })
+    it "depends on #lookup" do
+      instance.stub(:lookup).with("Hayoob!") { "yup" }
+      instance.wait_for("Hayoob!").should eq "yup"
+    end
+
+    it "waits for the service to appear" do
+      server = double(:uuid => 1, :ttl => 5, :services => ["Test"])
+
+      t = Thread.new do
+        sleep 0.5
+        instance.add_listing_for server
+      end
+
+      duration { instance.wait_for("Test") }.should be_within(0.01).of(0.5)
+      t.join
+    end
+
+    it "returns nil if the service doesn't appear withint the timeout period" do
+      server = double(:uuid => 1, :ttl => 5, :services => ["Test"])
+
+      t = Thread.new do
+        sleep 0.5
+        instance.add_listing_for server
+      end
 
-      instance.youngest_listing_for("Test").should be_nil
+      instance.wait_for("Test", 0.1).should be_nil
+      t.join
     end
   end
 

From 44857e626ec403837b51bb3c31d962bc03752439 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 5 Jun 2013 10:27:01 -0600
Subject: [PATCH 38/70] Require 'ostruct' before using it

---
 lib/protobuf/rpc/servers/zmq_runner.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/protobuf/rpc/servers/zmq_runner.rb b/lib/protobuf/rpc/servers/zmq_runner.rb
index f1219d9e..9911cc3c 100644
--- a/lib/protobuf/rpc/servers/zmq_runner.rb
+++ b/lib/protobuf/rpc/servers/zmq_runner.rb
@@ -1,3 +1,5 @@
+require 'ostruct'
+
 module Protobuf
   module Rpc
     class ZmqRunner

From b4b431d4d6eb0b564cfd66411e35df29425a54ec Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 5 Jun 2013 11:20:12 -0600
Subject: [PATCH 39/70] Include Thor::Actions in the CLI

---
 lib/protobuf/cli.rb | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index c1810fce..2a0ae23d 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -7,6 +7,8 @@
 
 module Protobuf
   class CLI < ::Thor
+    include ::Thor::Actions
+
     attr_accessor :runner, :mode
 
     default_task :start
@@ -168,7 +170,7 @@ def require_application!(app_file)
 
       # Loads protobuf in the given mode, exiting if somehow the mode is wrong.
       def require_protobuf!
-        require "protobuf/#{@mode}.rb"
+        require "protobuf/#{@mode}"
       rescue LoadError => e
         puts e.message, *(e.backtrace)
         say_and_exit!("Failed to load protobuf runner #{@mode}", e)

From 3f58484200c4d04ed449e3534eb5a8d7bff54f9f Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 5 Jun 2013 11:24:26 -0600
Subject: [PATCH 40/70] Require the server before creating the runner

---
 lib/protobuf/cli.rb | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index 2a0ae23d..5f634823 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -42,7 +42,6 @@ def start(app_file)
       configure_logger
       configure_traps
       configure_server_mode
-      require_protobuf!
       configure_gc
       configure_deprecation_warnings
 
@@ -168,14 +167,6 @@ def require_application!(app_file)
         say_and_exit!("Failed to load application file #{app_file}", e)
       end
 
-      # Loads protobuf in the given mode, exiting if somehow the mode is wrong.
-      def require_protobuf!
-        require "protobuf/#{@mode}"
-      rescue LoadError => e
-        puts e.message, *(e.backtrace)
-        say_and_exit!("Failed to load protobuf runner #{@mode}", e)
-      end
-
       def runner_options
         {
           :host => options.host,
@@ -204,16 +195,22 @@ def say_and_exit!(message, exception = nil)
       end
 
       def server_evented!
+        require 'protobuf/evented'
+
         @mode = :evented
         @runner = ::Protobuf::Rpc::EventedRunner.new(runner_options)
       end
 
       def server_socket!
+        require 'protobuf/socket'
+
         @mode = :socket
         @runner = ::Protobuf::Rpc::SocketRunner.new(runner_options)
       end
 
       def server_zmq!
+        require 'protobuf/zmq'
+
         @mode = :zmq
         @runner = ::Protobuf::Rpc::ZmqRunner.new(runner_options)
       end

From 914f5c59517cc399445a7e33d75a97ad72b84e4e Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 5 Jun 2013 11:39:53 -0600
Subject: [PATCH 41/70] Add CLI configuration for beacon address and port

---
 lib/protobuf/cli.rb | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index 5f634823..a1aec37c 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -30,6 +30,8 @@ class CLI < ::Thor
     option :zmq,                        :type => :boolean, :aliases => %w(-z), :desc => 'ZeroMQ Socket Mode for server and client connections.'
 
     option :broadcast_beacons,          :type => :boolean, :default => false, :desc => 'Broadcast beacons for dynamic discovery (Currently only available with ZeroMQ).'
+    option :beacon_address,             :type => :string,  :desc => 'Broadcast beacons to this address'
+    option :beacon_port,                :type => :string,  :desc => 'Broadcast beacons to this port'
     option :debug,                      :type => :boolean, :default => false, :aliases => %w(-d), :desc => 'Debug Mode. Override log level to DEBUG.'
     option :gc_pause_request,           :type => :boolean, :default => false, :desc => 'Enable/Disable GC pause during request.'
     option :print_deprecation_warnings, :type => :boolean, :default => nil, :desc => 'Cause use of deprecated fields to be printed or ignored.'
@@ -37,6 +39,8 @@ class CLI < ::Thor
     option :worker_port,                :type => :numeric, :default => nil, :desc => "Port for 'backend' where workers connect (defaults to port + 1)"
 
     def start(app_file)
+      $rpc_server = true
+
       debug_say 'Configuring the rpc_server process'
 
       configure_logger

From b5e378af4cc7fa3783f1687b6d9ccccd7792e6db Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 5 Jun 2013 11:48:57 -0600
Subject: [PATCH 42/70] Log heartbeat beacons

---
 lib/protobuf/rpc/servers/zmq/server.rb | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 90f0b7c2..99956a53 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -55,6 +55,10 @@ def beacon_port
           options[:beacon_port].to_i
         end
 
+        def beacon_uri
+          "udp://#{beacon_ip}:#{beacon_port}"
+        end
+
         def broadcast_beacons?
           !brokerless? && options[:broadcast_beacons]
         end
@@ -75,6 +79,8 @@ def broadcast_heartbeat
           )
 
           @beacon_socket.send heartbeat.serialize_to_string, 0
+
+          log_debug { sign_message("sent heartbeat to #{beacon_uri}") }
         end
 
         def brokerless?

From 6db51982d239bda053a52d126e10281fc43dd379 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 5 Jun 2013 11:52:57 -0600
Subject: [PATCH 43/70] Symbolize option keys in zmq server

---
 lib/protobuf/rpc/servers/zmq/server.rb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 99956a53..cf1dbd48 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -20,6 +20,9 @@ class Server
         attr_accessor :options
 
         def initialize(options)
+          # Symbolize keys
+          options.inject({}) {|h,(k,v)| h[k.to_sym] = v; h}
+
           @options = DEFAULT_OPTIONS.merge(options)
           @workers = []
 

From 7618f4d99ca6c6347cb40eceb53241b64bd0d3d8 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Thu, 6 Jun 2013 13:34:58 -0600
Subject: [PATCH 44/70] Fixes and improvements

---
 lib/protobuf/cli.rb                    | 19 +++++++------------
 lib/protobuf/rpc/connectors/zmq.rb     |  6 ++----
 lib/protobuf/rpc/servers/zmq/server.rb |  5 +----
 lib/protobuf/rpc/service.rb            |  7 +------
 lib/protobuf/rpc/service_directory.rb  |  2 --
 5 files changed, 11 insertions(+), 28 deletions(-)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index a1aec37c..819dd2f4 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -29,9 +29,9 @@ class CLI < ::Thor
     option :evented,                    :type => :boolean, :aliases => %w(-m), :desc => 'Evented Mode for server and client connections (uses EventMachine).'
     option :zmq,                        :type => :boolean, :aliases => %w(-z), :desc => 'ZeroMQ Socket Mode for server and client connections.'
 
-    option :broadcast_beacons,          :type => :boolean, :default => false, :desc => 'Broadcast beacons for dynamic discovery (Currently only available with ZeroMQ).'
+    option :broadcast_beacons,          :type => :boolean, :desc => 'Broadcast beacons for dynamic discovery (Currently only available with ZeroMQ).'
     option :beacon_address,             :type => :string,  :desc => 'Broadcast beacons to this address'
-    option :beacon_port,                :type => :string,  :desc => 'Broadcast beacons to this port'
+    option :beacon_port,                :type => :numeric, :desc => 'Broadcast beacons to this port'
     option :debug,                      :type => :boolean, :default => false, :aliases => %w(-d), :desc => 'Debug Mode. Override log level to DEBUG.'
     option :gc_pause_request,           :type => :boolean, :default => false, :desc => 'Enable/Disable GC pause during request.'
     option :print_deprecation_warnings, :type => :boolean, :default => nil, :desc => 'Cause use of deprecated fields to be printed or ignored.'
@@ -140,7 +140,7 @@ def configure_traps
 
           trap(signal) do
             @exit_requested = true
-            shutdown_server
+            shutdown_server!
           end
         end
       end
@@ -172,15 +172,10 @@ def require_application!(app_file)
       end
 
       def runner_options
-        {
-          :host => options.host,
-          :port => options.port,
-          :backlog => options.backlog,
-          :threshold => options.threshold,
-          :threads => options.threads,
-          :worker_port => options.worker_port || (options.port + 1),
-          :workers_only => !!ENV['PB_WORKERS_ONLY'] || options.workers_only
-        }
+        opt = options.inject({}){|h,(k,v)|h[k.to_sym]=v;h} # Symbolize keys
+        opt[:workers_only] = !!ENV['PB_WORKERS_ONLY'] || options.workers_only
+
+        opt
       end
 
       def say_and_exit!(message, exception = nil)
diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index 863f8bb5..004635c0 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -109,10 +109,8 @@ def poller
         def read_response
           return if error?
 
-          poller.readables.each do |readable|
-            @response_data = ''
-            zmq_error_check(readable.recv_string(@response_data), :socket_recv_string)
-          end
+          @response_data = ''
+          zmq_error_check(socket.recv_string(@response_data), :socket_recv_string)
 
           parse_response
         end
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index cf1dbd48..acfcfd6e 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -20,9 +20,6 @@ class Server
         attr_accessor :options
 
         def initialize(options)
-          # Symbolize keys
-          options.inject({}) {|h,(k,v)| h[k.to_sym] = v; h}
-
           @options = DEFAULT_OPTIONS.merge(options)
           @workers = []
 
@@ -39,7 +36,7 @@ def backend_ip
         end
 
         def backend_port
-          options[:worker_port]
+          options[:worker_port] || frontend_port + 1
         end
 
         def backend_uri
diff --git a/lib/protobuf/rpc/service.rb b/lib/protobuf/rpc/service.rb
index 50b221a6..c708de58 100644
--- a/lib/protobuf/rpc/service.rb
+++ b/lib/protobuf/rpc/service.rb
@@ -54,11 +54,6 @@ def self.host=(new_host)
         @_host = new_host
       end
 
-      def self.inherited(subclass)
-        @_subclasses ||= []
-        @_subclasses << subclass
-      end
-
       # Shorthand call to configure, passing a string formatted as hostname:port
       # e.g. 127.0.0.1:9933
       # e.g. localhost:0
@@ -104,7 +99,7 @@ def self.rpc_method?(name)
       # An array of defined service classes that contain implementation
       # code
       def self.implemented_services
-        classes = (@_subclasses || []).select do |subclass|
+        classes = (self.subclasses || []).select do |subclass|
           subclass.rpcs.any? do |(name, method)|
             subclass.method_defined? name
           end
diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index 66383827..346839ac 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -73,8 +73,6 @@ def initialize
 
       def add_listing_for(server)
         if server && server.uuid
-          log_debug { sign_message("Adding listing for server: #{server.inspect}") }
-
           @mutex.synchronize do
             @listings[server.uuid] = Listing.new(server)
           end

From 0e59c5dc4c1205c8720b485c46b183d52824ecb0 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Thu, 6 Jun 2013 13:59:11 -0600
Subject: [PATCH 45/70] Remove global

---
 lib/protobuf/cli.rb | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index 819dd2f4..fe8b0ca9 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -39,8 +39,6 @@ class CLI < ::Thor
     option :worker_port,                :type => :numeric, :default => nil, :desc => "Port for 'backend' where workers connect (defaults to port + 1)"
 
     def start(app_file)
-      $rpc_server = true
-
       debug_say 'Configuring the rpc_server process'
 
       configure_logger

From c4117ea47a4779c46f794d35cb3128f6256ce038 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Thu, 6 Jun 2013 14:23:12 -0600
Subject: [PATCH 46/70] Allow multiple service directories per machine

---
 lib/protobuf/rpc/service_directory.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index 346839ac..20ae6fd2 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -158,6 +158,7 @@ def wait_for(service, timeout = DEFAULT_TIMEOUT)
 
       def init_socket
         @socket = UDPSocket.new
+        @socket.setsockopt(:SOCKET, :REUSEADDR, true)
         @socket.bind(self.class.address, self.class.port)
       end
 

From b4cdec76d75fd94f5879bc08c6be9d10bdccafe3 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Thu, 6 Jun 2013 16:44:59 -0600
Subject: [PATCH 47/70] JRuby does not support shorthand sockopts

---
 lib/protobuf/rpc/servers/zmq/server.rb | 2 +-
 lib/protobuf/rpc/service_directory.rb  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index acfcfd6e..70de2b83 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -234,7 +234,7 @@ def wait_for_shutdown_signal
 
         def init_beacon_socket
           @beacon_socket = UDPSocket.new
-          @beacon_socket.setsockopt :SOCKET, :BROADCAST, true
+          @beacon_socket.setsockopt Socket::SOL_SOCKET, Socket::SO_BROADCAST, true
           @beacon_socket.connect beacon_ip, beacon_port
         end
 
diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index 20ae6fd2..8b5f8bb6 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -158,7 +158,7 @@ def wait_for(service, timeout = DEFAULT_TIMEOUT)
 
       def init_socket
         @socket = UDPSocket.new
-        @socket.setsockopt(:SOCKET, :REUSEADDR, true)
+        @socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, true)
         @socket.bind(self.class.address, self.class.port)
       end
 

From 208cc78c4b0fd6eaac32ea348e994601c66a994b Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Thu, 6 Jun 2013 16:52:08 -0600
Subject: [PATCH 48/70] Fix specs

---
 lib/protobuf/rpc/servers/zmq/server.rb | 2 +-
 lib/protobuf/rpc/service_directory.rb  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 70de2b83..0b22467f 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -234,7 +234,7 @@ def wait_for_shutdown_signal
 
         def init_beacon_socket
           @beacon_socket = UDPSocket.new
-          @beacon_socket.setsockopt Socket::SOL_SOCKET, Socket::SO_BROADCAST, true
+          @beacon_socket.setsockopt ::Socket::SOL_SOCKET, ::Socket::SO_BROADCAST, true
           @beacon_socket.connect beacon_ip, beacon_port
         end
 
diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index 8b5f8bb6..fad13791 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -158,7 +158,7 @@ def wait_for(service, timeout = DEFAULT_TIMEOUT)
 
       def init_socket
         @socket = UDPSocket.new
-        @socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, true)
+        @socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEADDR, true)
         @socket.bind(self.class.address, self.class.port)
       end
 

From 598b39b575dcbf536c0ebcd7575157a2a6f06c2f Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 7 Jun 2013 14:13:23 -0600
Subject: [PATCH 49/70] Default beacons to directories port and address

---
 lib/protobuf/cli.rb                    |  5 +++--
 lib/protobuf/rpc/servers/zmq/server.rb | 22 +++++++++++++++++-----
 lib/protobuf/rpc/service_directory.rb  |  6 +++---
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index fe8b0ca9..4d96f7e3 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -29,9 +29,10 @@ class CLI < ::Thor
     option :evented,                    :type => :boolean, :aliases => %w(-m), :desc => 'Evented Mode for server and client connections (uses EventMachine).'
     option :zmq,                        :type => :boolean, :aliases => %w(-z), :desc => 'ZeroMQ Socket Mode for server and client connections.'
 
+    option :beacon_address,             :type => :string,  :desc => 'Broadcast beacons to this address (defaul: value of ServiceDirectory.address)'
+    option :beacon_interval,            :type => :numeric, :desc => 'Broadcast beacons every N seconds. (default: 5)'
+    option :beacon_port,                :type => :numeric, :desc => 'Broadcast beacons to this port (default: value of ServiceDirectory.port)'
     option :broadcast_beacons,          :type => :boolean, :desc => 'Broadcast beacons for dynamic discovery (Currently only available with ZeroMQ).'
-    option :beacon_address,             :type => :string,  :desc => 'Broadcast beacons to this address'
-    option :beacon_port,                :type => :numeric, :desc => 'Broadcast beacons to this port'
     option :debug,                      :type => :boolean, :default => false, :aliases => %w(-d), :desc => 'Debug Mode. Override log level to DEBUG.'
     option :gc_pause_request,           :type => :boolean, :default => false, :desc => 'Enable/Disable GC pause during request.'
     option :print_deprecation_warnings, :type => :boolean, :default => nil, :desc => 'Cause use of deprecated fields to be printed or ignored.'
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 0b22467f..bd2bf23e 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -11,9 +11,7 @@ class Server
         include ::Protobuf::Rpc::Zmq::Util
 
         DEFAULT_OPTIONS = {
-          :beacon_address => "255.255.255.255",
           :beacon_interval => 5,
-          :beacon_port => 9398,
           :broadcast_beacons => false
         }
 
@@ -48,11 +46,25 @@ def beacon_interval
         end
 
         def beacon_ip
-          @beacon_ip ||= resolve_ip(options[:beacon_address])
+          unless @beacon_ip
+            unless address = options[:beacon_address]
+              address = ::Protobuf::Rpc::ServiceDirectory.address
+            end
+
+            @beacon_ip = resolve_ip(address)
+          end
+
+          @beacon_ip
         end
 
         def beacon_port
-          options[:beacon_port].to_i
+          unless @beacon_port
+            unless port = options[:beacon_port]
+              port = ::Protobuf::Rpc::ServiceDirectory.port
+            end
+
+            @beacon_port = port.to_i
+          end
         end
 
         def beacon_uri
@@ -192,7 +204,7 @@ def to_proto
             :uuid => uuid,
             :address => frontend_ip,
             :port => frontend_port.to_s,
-            :ttl => beacon_interval * 3,
+            :ttl => (beacon_interval * 1.5).ceil,
             :services => ::Protobuf::Rpc::Service.implemented_services
           )
         end
diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index fad13791..178bf5b7 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -12,8 +12,8 @@ class ServiceDirectory
       include ::Singleton
       include ::Protobuf::Logger::LogMethods
 
-      DEFAULT_ADDRESS = "0.0.0.0"
-      DEFAULT_PORT = 9398
+      DEFAULT_ADDRESS = "255.255.255.255"
+      DEFAULT_PORT = 53000
       DEFAULT_TIMEOUT = 1
 
       class Listing < Delegator
@@ -52,7 +52,7 @@ def self.address
       end
 
       def self.port
-        @port ||= DEFAULT_PORT
+        (@port ||= DEFAULT_PORT).to_i
       end
 
       def self.start

From 13699c39d0bc9d6fec2075c4d528c7e530e464cc Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 7 Jun 2013 14:48:12 -0600
Subject: [PATCH 50/70] It helps to return stuffs

---
 lib/protobuf/rpc/servers/zmq/server.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index bd2bf23e..8aed3e6b 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -65,6 +65,8 @@ def beacon_port
 
             @beacon_port = port.to_i
           end
+
+          @beacon_port
         end
 
         def beacon_uri

From 38e773fbe22ffd67308595d571a52cb6961bcb0b Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 7 Jun 2013 15:21:28 -0600
Subject: [PATCH 51/70] Shutdown the service directory if it's running

---
 lib/protobuf/cli.rb | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index 4d96f7e3..87d639ee 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -216,6 +216,7 @@ def server_zmq!
       def shutdown_server!
         ::Protobuf::Logger.info { 'RPC Server shutting down...' }
         @runner.try :stop
+        ::Protobuf::Rpc::ServiceDirectory.instance.stop
         ::Protobuf::Logger.info { 'Shutdown complete' }
       end
 

From 84094c0ca3f517c3e2a85c501caf3f9bff4fa754 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 7 Jun 2013 15:57:02 -0600
Subject: [PATCH 52/70] Defer server creation

This gives the application time to load and make any needed
configuration changes.
---
 lib/protobuf/rpc/servers/zmq_runner.rb | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq_runner.rb b/lib/protobuf/rpc/servers/zmq_runner.rb
index 9911cc3c..656c634a 100644
--- a/lib/protobuf/rpc/servers/zmq_runner.rb
+++ b/lib/protobuf/rpc/servers/zmq_runner.rb
@@ -15,21 +15,21 @@ def initialize(options)
                      raise "Cannot parser Zmq Server - server options"
                    end
 
-        @server = ::Protobuf::Rpc::Zmq::Server.new(@options)
       end
 
       def run
+        @server = ::Protobuf::Rpc::Zmq::Server.new(@options)
         register_signals
         yield if block_given?
         @server.run
       end
 
       def running?
-        @server.running?
+        @server.try :running?
       end
 
       def stop
-        @server.stop
+        @server.try :stop
       end
 
       private

From c9ca7e31952b13668372dfaff5794a032a577856 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 7 Jun 2013 15:59:23 -0600
Subject: [PATCH 53/70] Make the caller responsible for casting

---
 lib/protobuf/rpc/service_directory.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index 178bf5b7..44222e5f 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -52,7 +52,7 @@ def self.address
       end
 
       def self.port
-        (@port ||= DEFAULT_PORT).to_i
+        @port ||= DEFAULT_PORT
       end
 
       def self.start
@@ -159,7 +159,7 @@ def wait_for(service, timeout = DEFAULT_TIMEOUT)
       def init_socket
         @socket = UDPSocket.new
         @socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_REUSEADDR, true)
-        @socket.bind(self.class.address, self.class.port)
+        @socket.bind(self.class.address, self.class.port.to_i)
       end
 
       def run

From 6b365f1ccbf170ae030bf8d7e687154e7ff1f84f Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 7 Jun 2013 16:00:32 -0600
Subject: [PATCH 54/70] Improve service directory logging

---
 lib/protobuf/rpc/service_directory.rb | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index 44222e5f..e1c7576d 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -73,6 +73,12 @@ def initialize
 
       def add_listing_for(server)
         if server && server.uuid
+
+          log_debug do
+            action = @listings[server.uuid] ? "Updating" : "Adding";
+            sign_message("#{action} server: #{server.inspect}")
+          end
+
           @mutex.synchronize do
             @listings[server.uuid] = Listing.new(server)
           end
@@ -123,8 +129,8 @@ def running?
 
       def start
         unless running?
-          log_debug { sign_message("starting") }
           init_socket
+          log_debug { sign_message("listining at udp://#{self.class.address}:#{self.class.port}") }
           @thread = Thread.new { self.send(:run) }
         end
 
@@ -168,8 +174,6 @@ def run
           beacon = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new
           beacon.parse_from_string(data) rescue nil
 
-          log_debug { sign_message("received beacon: #{beacon.inspect} from #{addr}") }
-
           case beacon.beacon_type
           when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
             add_listing_for(beacon.server)

From 135fae42e292b8f1e01c4d0486839f584c4836e7 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 7 Jun 2013 17:51:53 -0600
Subject: [PATCH 55/70] Fix typo

---
 lib/protobuf/rpc/service_directory.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index e1c7576d..ab0c6f0b 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -130,7 +130,7 @@ def running?
       def start
         unless running?
           init_socket
-          log_debug { sign_message("listining at udp://#{self.class.address}:#{self.class.port}") }
+          log_debug { sign_message("listening to udp://#{self.class.address}:#{self.class.port}") }
           @thread = Thread.new { self.send(:run) }
         end
 

From a2956b3b23489e2762602b245a67565086531001 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 7 Jun 2013 18:11:39 -0600
Subject: [PATCH 56/70] Only lookup the server_uri once per connection

---
 lib/protobuf/rpc/connectors/zmq.rb | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index 004635c0..0bb6a4ea 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -58,11 +58,12 @@ def close_connection
         def connect_to_rpc_server
           return if error?
 
-          log_debug { sign_message("Establishing connection: #{server_uri}") }
+          uri = server_uri
+          log_debug { sign_message("Establishing connection: #{uri}") }
           socket.setsockopt(::ZMQ::LINGER, 0)
-          zmq_error_check(socket.connect(server_uri), :socket_connect)
+          zmq_error_check(socket.connect(uri), :socket_connect)
           zmq_error_check(poller.register_readable(socket), :poller_register_readable)
-          log_debug { sign_message("Connection established to #{server_uri}") }
+          log_debug { sign_message("Connection established to #{uri}") }
         end
 
         # Method to determine error state, must be used with Connector API.

From 2161263f811ac68ac34bad6b330860559b7f8998 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 7 Jun 2013 18:18:26 -0600
Subject: [PATCH 57/70] Add a restart method to the service directory

This is useful for apps running unicorn. Call restart in the after_fork
callback.
---
 lib/protobuf/rpc/service_directory.rb | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index ab0c6f0b..5b957679 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -123,6 +123,11 @@ def remove_listing_for(server)
         end
       end
 
+      def restart
+        stop
+        start
+      end
+
       def running?
         !!@thread.try(:alive?)
       end

From 60c17aee23b6ab3fa25dfcccc45cb1ce2e85d967 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Fri, 7 Jun 2013 18:23:40 -0600
Subject: [PATCH 58/70] Favor the address captured by the socket

---
 lib/protobuf/rpc/service_directory.rb | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index 5b957679..a63041be 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -179,6 +179,9 @@ def run
           beacon = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new
           beacon.parse_from_string(data) rescue nil
 
+          # Favor the address captured by the socket
+          beacon.try(:server).try(:address=, addr[3])
+
           case beacon.beacon_type
           when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
             add_listing_for(beacon.server)

From cff848089d6a2e8cf6243353099edfd6d70c47f5 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Mon, 10 Jun 2013 11:28:19 -0600
Subject: [PATCH 59/70] Rename server_uri to lookup_server_uri

---
 lib/protobuf/rpc/connectors/zmq.rb | 38 +++++++++++++++---------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/lib/protobuf/rpc/connectors/zmq.rb b/lib/protobuf/rpc/connectors/zmq.rb
index 0bb6a4ea..ac221164 100644
--- a/lib/protobuf/rpc/connectors/zmq.rb
+++ b/lib/protobuf/rpc/connectors/zmq.rb
@@ -58,12 +58,12 @@ def close_connection
         def connect_to_rpc_server
           return if error?
 
-          uri = server_uri
-          log_debug { sign_message("Establishing connection: #{uri}") }
+          server_uri = lookup_server_uri
+          log_debug { sign_message("Establishing connection: #{server_uri}") }
           socket.setsockopt(::ZMQ::LINGER, 0)
-          zmq_error_check(socket.connect(uri), :socket_connect)
+          zmq_error_check(socket.connect(server_uri), :socket_connect)
           zmq_error_check(poller.register_readable(socket), :poller_register_readable)
-          log_debug { sign_message("Connection established to #{uri}") }
+          log_debug { sign_message("Connection established to #{server_uri}") }
         end
 
         # Method to determine error state, must be used with Connector API.
@@ -72,6 +72,21 @@ def error?
           !! @error
         end
 
+        # Lookup a server uri for the requested service in the service
+        # directory. If the service directory is not running, default
+        # to the host and port in the options
+        #
+        def lookup_server_uri
+          if service_directory.running?
+            listing = service_directory.lookup(service)
+            host, port = listing.address, listing.port if listing
+          end
+
+          host, port = options[:host], options[:port] unless host && port
+
+          "tcp://#{host}:#{port}"
+        end
+
         # Trying a number of times, attempt to get a response from the server.
         # If we haven't received a legitimate response in the CLIENT_RETRIES number
         # of retries, fail the request.
@@ -125,21 +140,6 @@ def send_data
           zmq_error_check(socket.send_string(@request_data), :socket_send_string)
         end
 
-        # The location of the server. If the ServiceDirectory is running then
-        # we ask it for a server that hosts the requested service. If it is
-        # not running, or it cannot be found, we fallback to the options.
-        #
-        def server_uri
-          if service_directory.running?
-            listing = service_directory.lookup(service)
-            host, port = listing.address, listing.port if listing
-          end
-
-          host, port = options[:host], options[:port] unless host && port
-
-          "tcp://#{host}:#{port}"
-        end
-
         # The service we're attempting to connect to
         #
         def service

From 7eb00e06dcd1b77cdf8da884c71d2ae7f6e8c4cd Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Mon, 10 Jun 2013 11:29:37 -0600
Subject: [PATCH 60/70] Reduce default timeout from 5 min to 15 sec

---
 lib/protobuf/rpc/connectors/base.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/protobuf/rpc/connectors/base.rb b/lib/protobuf/rpc/connectors/base.rb
index e00c3889..eb7a2f52 100644
--- a/lib/protobuf/rpc/connectors/base.rb
+++ b/lib/protobuf/rpc/connectors/base.rb
@@ -17,7 +17,7 @@ module Connectors
         :request        => nil,           # The request object sent by the client
         :request_type   => nil,           # The request type expected by the client
         :response_type  => nil,           # The response type expected by the client
-        :timeout        => 300,           # The default timeout for the request, also handled by client.rb
+        :timeout        => 15,            # The default timeout for the request, also handled by client.rb
         :client_host    => nil            # The hostname or address of this client
       }
 

From 8c1d1e29927121a7a5dd20ad33b87b8dd4d16849 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Mon, 10 Jun 2013 11:34:15 -0600
Subject: [PATCH 61/70] Update logging

---
 lib/protobuf/rpc/service_directory.rb | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index a63041be..0a36d804 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -84,13 +84,11 @@ def add_listing_for(server)
           end
 
         else
-          log_debug { sign_message("Cannot add server without uuid: #{server.inspect}") }
+          log_info { sign_message("Cannot add server without uuid: #{server.inspect}") }
         end
       end
 
       def lookup(service)
-        log_debug { sign_message("lookup #{service}") }
-
         @mutex.synchronize do
           listings = @listings.values.select do |listing|
             listing.services.any? do |listed_service|
@@ -119,7 +117,7 @@ def remove_listing_for(server)
           end
 
         else
-          log_debug { sign_message("Cannot remove server without uuid: #{server.inspect}") }
+          log_info { sign_message("Cannot remove server without uuid: #{server.inspect}") }
         end
       end
 
@@ -135,7 +133,7 @@ def running?
       def start
         unless running?
           init_socket
-          log_debug { sign_message("listening to udp://#{self.class.address}:#{self.class.port}") }
+          log_info { sign_message("listening to udp://#{self.class.address}:#{self.class.port}") }
           @thread = Thread.new { self.send(:run) }
         end
 
@@ -143,7 +141,7 @@ def start
       end
 
       def stop
-        log_debug { sign_message("stopping") }
+        log_info { sign_message("Stopping directory") }
 
         @mutex.synchronize do
           @thread.try(:kill)
@@ -162,6 +160,7 @@ def wait_for(service, timeout = DEFAULT_TIMEOUT)
           listing
         end
       rescue
+        log_info { sign_message("no listing found for #{service}") }
         nil
       end
 

From 09154fd9ec0429a2a170f5a461ab8e629ad1650e Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Mon, 10 Jun 2013 17:36:45 -0600
Subject: [PATCH 62/70] Fix whitespace

---
 lib/protobuf/rpc/error/client_error.rb | 10 +++++-----
 lib/protobuf/rpc/error/server_error.rb | 14 +++++++-------
 2 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/lib/protobuf/rpc/error/client_error.rb b/lib/protobuf/rpc/error/client_error.rb
index 5b65ed3b..19d161af 100644
--- a/lib/protobuf/rpc/error/client_error.rb
+++ b/lib/protobuf/rpc/error/client_error.rb
@@ -2,30 +2,30 @@
 
 module Protobuf
   module Rpc
-    
+
     class InvalidRequestProto < PbError
       def initialize(message='Invalid request type given')
         super message, 'INVALID_REQUEST_PROTO'
       end
     end
-    
+
     class BadResponseProto < PbError
       def initialize(message='Bad response type from server')
         super message, 'BAD_RESPONSE_PROTO'
       end
     end
-    
+
     class UnkownHost < PbError
       def initialize(message='Unknown host or port')
         super message, 'UNKNOWN_HOST'
       end
     end
-    
+
     class IOError < PbError
       def initialize(message='IO Error occurred')
         super message, 'IO_ERROR'
       end
     end
-    
+
   end
 end
diff --git a/lib/protobuf/rpc/error/server_error.rb b/lib/protobuf/rpc/error/server_error.rb
index f0191776..a37f0a37 100644
--- a/lib/protobuf/rpc/error/server_error.rb
+++ b/lib/protobuf/rpc/error/server_error.rb
@@ -2,42 +2,42 @@
 
 module Protobuf
   module Rpc
-    
+
     class BadRequestData < PbError
       def initialize message='Unable to parse request'
         super message, 'BAD_REQUEST_DATA'
       end
     end
-    
+
     class BadRequestProto < PbError
       def initialize message='Request is of wrong type'
         super message, 'BAD_REQUEST_PROTO'
       end
     end
-    
+
     class ServiceNotFound < PbError
       def initialize message='Service class not found'
         super message, 'SERVICE_NOT_FOUND'
       end
     end
-    
+
     class MethodNotFound < PbError
       def initialize message='Service method not found'
         super message, 'METHOD_NOT_FOUND'
       end
     end
-    
+
     class RpcError < PbError
       def initialize message='RPC exception occurred'
         super message, 'RPC_ERROR'
       end
     end
-    
+
     class RpcFailed < PbError
       def initialize message='RPC failed'
         super message, 'RPC_FAILED'
       end
     end
-    
+
   end
 end

From 18f71258e13c90b1c71033a643d468171fa4d9a8 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Tue, 11 Jun 2013 10:21:10 -0600
Subject: [PATCH 63/70] Reconcile with comments from the pull request

---
 lib/protobuf/rpc/servers/zmq/broker.rb        | 63 ++++++++++++-------
 lib/protobuf/rpc/servers/zmq/server.rb        | 18 +++---
 lib/protobuf/rpc/servers/zmq/worker.rb        | 46 ++++++++------
 spec/lib/protobuf/rpc/connectors/zmq_spec.rb  |  8 +--
 .../protobuf/rpc/service_directory_spec.rb    |  4 +-
 5 files changed, 82 insertions(+), 57 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq/broker.rb b/lib/protobuf/rpc/servers/zmq/broker.rb
index 2aae9006..0e339722 100644
--- a/lib/protobuf/rpc/servers/zmq/broker.rb
+++ b/lib/protobuf/rpc/servers/zmq/broker.rb
@@ -19,7 +19,19 @@ def initialize(server)
         end
 
         def join
-          @thread.try :join
+          @thread.try(:join)
+        end
+
+        def read_from_backend
+          [].tap do |frames|
+            zmq_error_check(@backend_socket.recv_strings(frames))
+          end
+        end
+
+        def read_from_frontend
+          [].tap do |frames|
+            zmq_error_check(@frontend_socket.recv_strings(frames))
+          end
         end
 
         def run
@@ -31,15 +43,14 @@ def run
                 case readable
                 when @frontend_socket
                   if idle_workers.any?
-                    zmq_error_check(@frontend_socket.recv_strings frames = [])
-                    frames.unshift(idle_workers.shift, "")
-                    zmq_error_check(@backend_socket.send_strings frames)
+                    frames = read_from_frontend
+                    write_to_backend([idle_workers.shift, ""] + frames)
                   end
                 when @backend_socket
-                  zmq_error_check(@backend_socket.recv_strings frames = [])
-                  idle_workers << frames.shift(2)[0]
+                  worker, ignore, *frames = read_from_backend
+                  idle_workers << worker
                   unless frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
-                    zmq_error_check(@frontend_socket.send_strings frames)
+                    write_to_frontend(frames)
                   end
                 when @shutdown_socket
                   throw :shutdown
@@ -51,6 +62,14 @@ def run
           teardown
         end
 
+        def write_to_backend(frames)
+          zmq_error_check(@backend_socket.send_strings(frames))
+        end
+
+        def write_to_frontend(frames)
+          zmq_error_check(@frontend_socket.send_strings(frames))
+        end
+
         def start
           log_debug { sign_message("starting broker") }
 
@@ -66,41 +85,41 @@ def shutdown_uri
         end
 
         def signal_shutdown
-          socket = @zmq_context.socket ZMQ::PAIR
-          zmq_error_check(socket.connect shutdown_uri)
+          socket = @zmq_context.socket(ZMQ::PAIR)
+          zmq_error_check(socket.connect(shutdown_uri))
           zmq_error_check(socket.send_string ".")
           zmq_error_check(socket.close)
         end
 
         def teardown
-          @frontend_socket.try :close
-          @backend_socket.try :close
-          @shutdown_socket.try :close
-          @zmq_context.try :terminate
+          @frontend_socket.try(:close)
+          @backend_socket.try(:close)
+          @shutdown_socket.try(:close)
+          @zmq_context.try(:terminate)
         end
 
         private
 
         def init_backend_socket
-          @backend_socket = @zmq_context.socket ZMQ::ROUTER
-          zmq_error_check(@backend_socket.bind @server.backend_uri)
+          @backend_socket = @zmq_context.socket(ZMQ::ROUTER)
+          zmq_error_check(@backend_socket.bind(@server.backend_uri))
         end
 
         def init_frontend_socket
-          @frontend_socket = @zmq_context.socket ZMQ::ROUTER
-          zmq_error_check(@frontend_socket.bind @server.frontend_uri)
+          @frontend_socket = @zmq_context.socket(ZMQ::ROUTER)
+          zmq_error_check(@frontend_socket.bind(@server.frontend_uri))
         end
 
         def init_poller
           @poller = ZMQ::Poller.new
-          @poller.register_readable @frontend_socket
-          @poller.register_readable @backend_socket
-          @poller.register_readable @shutdown_socket
+          @poller.register_readable(@frontend_socket)
+          @poller.register_readable(@backend_socket)
+          @poller.register_readable(@shutdown_socket)
         end
 
         def init_shutdown_socket
-          @shutdown_socket = @zmq_context.socket ZMQ::PAIR
-          zmq_error_check(@shutdown_socket.bind shutdown_uri)
+          @shutdown_socket = @zmq_context.socket(ZMQ::PAIR)
+          zmq_error_check(@shutdown_socket.bind(shutdown_uri))
         end
 
         def init_zmq_context
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 8aed3e6b..16638847 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -92,7 +92,7 @@ def broadcast_heartbeat
             :server => self.to_proto
           )
 
-          @beacon_socket.send heartbeat.serialize_to_string, 0
+          @beacon_socket.send(heartbeat.serialize_to_string, 0)
 
           log_debug { sign_message("sent heartbeat to #{beacon_uri}") }
         end
@@ -187,14 +187,14 @@ def stop_broker
         end
 
         def stop_workers
-          @workers.each &:signal_shutdown
+          @workers.each(&:signal_shutdown)
           Thread.pass until reap_dead_workers.empty?
         end
 
         def teardown
-          @shutdown_socket.try :close
-          @beacon_socket.try :close
-          @zmq_context.try :terminate
+          @shutdown_socket.try(:close)
+          @beacon_socket.try(:close)
+          @zmq_context.try(:terminate)
         end
 
         def total_workers
@@ -222,7 +222,7 @@ def wait_for_shutdown_signal
           next_reaping = time + reaping_interval
           next_cycle = time + maintenance_interval
           poller = ZMQ::Poller.new
-          poller.register_readable @shutdown_socket
+          poller.register_readable(@shutdown_socket)
 
           # If the poller returns 1, a shutdown signal has been received.
           # If the poller returns -1, something went wrong.
@@ -248,12 +248,12 @@ def wait_for_shutdown_signal
 
         def init_beacon_socket
           @beacon_socket = UDPSocket.new
-          @beacon_socket.setsockopt ::Socket::SOL_SOCKET, ::Socket::SO_BROADCAST, true
-          @beacon_socket.connect beacon_ip, beacon_port
+          @beacon_socket.setsockopt(::Socket::SOL_SOCKET, ::Socket::SO_BROADCAST, true)
+          @beacon_socket.connect(beacon_ip, beacon_port)
         end
 
         def init_shutdown_socket
-          @shutdown_socket = @zmq_context.socket ZMQ::PAIR
+          @shutdown_socket = @zmq_context.socket(ZMQ::PAIR)
           zmq_error_check(@shutdown_socket.bind shutdown_uri)
         end
 
diff --git a/lib/protobuf/rpc/servers/zmq/worker.rb b/lib/protobuf/rpc/servers/zmq/worker.rb
index 89ac5f46..b648e4fe 100644
--- a/lib/protobuf/rpc/servers/zmq/worker.rb
+++ b/lib/protobuf/rpc/servers/zmq/worker.rb
@@ -29,13 +29,11 @@ def alive?
         end
 
         def join
-          @thread.try :join
+          @thread.try(:join)
         end
 
         def process_request
-          zmq_error_check(@backend_socket.recv_strings(frames = []))
-
-          @client_address, empty, @request_data = *frames
+          @client_address, empty, @request_data = read_from_backend
 
           unless @request_data.nil?
             log_debug { sign_message("handling request") }
@@ -43,13 +41,19 @@ def process_request
           end
         end
 
+        def read_from_backend
+          [].tap do |frames|
+            zmq_error_check(@backend_socket.recv_strings(frames))
+          end
+        end
+
         def run
           poller = ::ZMQ::Poller.new
           poller.register_readable(@backend_socket)
           poller.register_readable(@shutdown_socket)
 
           # Send request to broker telling it we are ready
-          zmq_error_check(@backend_socket.send_string(::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE))
+          write_to_backend([::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE])
 
           catch(:shutdown) do
             while poller.poll > 0
@@ -69,13 +73,11 @@ def run
         end
 
         def send_data
-          response_data = @response.to_s # to_s is aliases as serialize_to_string in Message
-
-          frames = [@client_address, "", response_data]
+          data = @response.serialize_to_string
 
-          @stats.response_size = response_data.size
+          @stats.response_size = data.size
 
-          zmq_error_check(@backend_socket.send_strings(frames))
+          write_to_backend([@client_address, "", data])
         end
 
         def shutdown_uri
@@ -84,8 +86,8 @@ def shutdown_uri
 
         def signal_shutdown
           socket = @zmq_context.socket ZMQ::PAIR
-          zmq_error_check(socket.connect shutdown_uri)
-          zmq_error_check(socket.send_string ".")
+          zmq_error_check(socket.connect(shutdown_uri))
+          zmq_error_check(socket.send_string("."))
           zmq_error_check(socket.close)
         end
 
@@ -95,7 +97,7 @@ def start
               self.run
             rescue => e
               message = "Worker failed: #{e.inspect}\n #{e.backtrace.join($/)}"
-              $stderr.puts message
+              $stderr.puts(message)
               log_error { message }
             end
           end
@@ -104,9 +106,13 @@ def start
         end
 
         def teardown
-          @backend_socket.try :close
-          @shutdown_socket.try :close
-          @zmq_context.try :terminate
+          @backend_socket.try(:close)
+          @shutdown_socket.try(:close)
+          @zmq_context.try(:terminate)
+        end
+
+        def write_to_backend(frames)
+          zmq_error_check(@backend_socket.send_strings(frames))
         end
 
         private
@@ -116,13 +122,13 @@ def init_zmq_context
         end
 
         def init_backend_socket
-          @backend_socket = @zmq_context.socket ZMQ::REQ
-          zmq_error_check(@backend_socket.connect @server.backend_uri)
+          @backend_socket = @zmq_context.socket(ZMQ::REQ)
+          zmq_error_check(@backend_socket.connect(@server.backend_uri))
         end
 
         def init_shutdown_socket
-          @shutdown_socket = @zmq_context.socket ZMQ::PAIR
-          zmq_error_check(@shutdown_socket.bind shutdown_uri)
+          @shutdown_socket = @zmq_context.socket(ZMQ::PAIR)
+          zmq_error_check(@shutdown_socket.bind(shutdown_uri))
         end
       end
     end
diff --git a/spec/lib/protobuf/rpc/connectors/zmq_spec.rb b/spec/lib/protobuf/rpc/connectors/zmq_spec.rb
index 7e32c41e..2fd0afb9 100644
--- a/spec/lib/protobuf/rpc/connectors/zmq_spec.rb
+++ b/spec/lib/protobuf/rpc/connectors/zmq_spec.rb
@@ -28,7 +28,7 @@
     ::ZMQ::Context.stub(:new).and_return(zmq_context_mock)
   end
 
-  describe "#server_uri" do
+  describe "#lookup_server_uri" do
     let(:service_directory) { double('ServiceDirectory', :running? => running? ) }
     let(:listing) { double('Listing', :address => '127.0.0.2', :port => 9399) }
     let(:running?) { true }
@@ -40,12 +40,12 @@
     context "when the service directory is running" do
       it "searches the service directory" do
         service_directory.should_receive(:lookup).and_return(listing)
-        subject.send(:server_uri).should eq "tcp://127.0.0.2:9399"
+        subject.send(:lookup_server_uri).should eq "tcp://127.0.0.2:9399"
       end
 
       it "defaults to the options" do
         service_directory.should_receive(:lookup) { nil }
-        subject.send(:server_uri).should eq "tcp://127.0.0.1:9400"
+        subject.send(:lookup_server_uri).should eq "tcp://127.0.0.1:9400"
       end
     end
 
@@ -54,7 +54,7 @@
 
       it "does not search the directory" do
         service_directory.should_not_receive(:lookup)
-        subject.send(:server_uri).should eq "tcp://127.0.0.1:9400"
+        subject.send(:lookup_server_uri).should eq "tcp://127.0.0.1:9400"
       end
     end
 
diff --git a/spec/lib/protobuf/rpc/service_directory_spec.rb b/spec/lib/protobuf/rpc/service_directory_spec.rb
index 31d41e49..e4c858d8 100644
--- a/spec/lib/protobuf/rpc/service_directory_spec.rb
+++ b/spec/lib/protobuf/rpc/service_directory_spec.rb
@@ -149,7 +149,7 @@ def thread
     end
 
     context "receiving a heartbeat" do
-      let(:server) { ::Protobuf::Rpc::DynamicDiscovery::Server.new(:uuid => 'heartbeat') }
+      let(:server) { ::Protobuf::Rpc::DynamicDiscovery::Server.new(:uuid => 'heartbeat', :address => '127.0.0.1') }
       let(:beacon) { ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
         :server => server,
         :beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
@@ -165,7 +165,7 @@ def thread
     end
 
     context "receiving a flatline" do
-      let(:server) { ::Protobuf::Rpc::DynamicDiscovery::Server.new(:uuid => 'flatline') }
+      let(:server) { ::Protobuf::Rpc::DynamicDiscovery::Server.new(:uuid => 'flatline', :address => '127.0.0.1') }
       let(:beacon) { ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
         :server => server,
         :beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE

From 949eef66eb74e38b4d1908dae205f1a8790a17e7 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Tue, 11 Jun 2013 10:30:02 -0600
Subject: [PATCH 64/70] Reconcile with comments from the pull request

---
 lib/protobuf/rpc/servers/socket/server.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/protobuf/rpc/servers/socket/server.rb b/lib/protobuf/rpc/servers/socket/server.rb
index 94a38dfa..50298ad3 100644
--- a/lib/protobuf/rpc/servers/socket/server.rb
+++ b/lib/protobuf/rpc/servers/socket/server.rb
@@ -107,7 +107,7 @@ def running?
 
         def stop
           @running = false
-          @server.try :close
+          @server.try(:close)
         end
       end
     end

From bbab1cb143ed5c139c83fc75083d3fa8c800b15c Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Tue, 11 Jun 2013 12:24:51 -0600
Subject: [PATCH 65/70] Cleanup

---
 lib/protobuf/cli.rb | 95 ++++++++++++++++++++++++++-------------------
 1 file changed, 55 insertions(+), 40 deletions(-)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index 87d639ee..3ef067c9 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -40,24 +40,25 @@ class CLI < ::Thor
     option :worker_port,                :type => :numeric, :default => nil, :desc => "Port for 'backend' where workers connect (defaults to port + 1)"
 
     def start(app_file)
-      debug_say 'Configuring the rpc_server process'
+      debug_say('Configuring the rpc_server process')
 
       configure_logger
       configure_traps
-      configure_server_mode
+      configure_runner_mode
+      create_runner
+      configure_process_name(app_file)
       configure_gc
       configure_deprecation_warnings
 
-      require_application!(app_file) unless exit_requested?
-      configure_process_name(app_file) unless exit_requested?
-      start_server! unless exit_requested?
+      require_application(app_file) unless exit_requested?
+      start_server unless exit_requested?
     rescue => e
-      say_and_exit!('ERROR: RPC Server failed to start.', e)
+      say_and_exit('ERROR: RPC Server failed to start.', e)
     end
 
     desc 'version', 'Print ruby and protoc versions and exit.'
     def version
-      say "Ruby Protobuf v#{::Protobuf::VERSION}, protoc v#{::Protobuf::PROTOC_VERSION}"
+      say("Ruby Protobuf v#{::Protobuf::VERSION}, protoc v#{::Protobuf::PROTOC_VERSION}")
     end
 
     no_tasks do
@@ -73,7 +74,7 @@ def configure_deprecation_warnings
 
       # If we pause during request we don't need to pause in serialization
       def configure_gc
-        debug_say 'Configuring gc'
+        debug_say('Configuring gc')
 
         if defined?(JRUBY_VERSION)
           # GC.enable/disable are noop's on Jruby
@@ -85,7 +86,7 @@ def configure_gc
 
       # Setup the protobuf logger.
       def configure_logger
-        debug_say 'Configuring logger'
+        debug_say('Configuring logger')
         ::Protobuf::Logger.configure({ :file => options.log || STDOUT,
                                        :level => options.debug? ? ::Logger::DEBUG : options.level })
 
@@ -96,32 +97,32 @@ def configure_logger
 
       # Re-write the $0 var to have a nice process name in ps.
       def configure_process_name(app_file)
-        debug_say 'Configuring process name'
-        $0 = "rpc_server --#{@mode} #{options.host}:#{options.port} #{app_file}"
+        debug_say('Configuring process name')
+        $0 = "rpc_server --#{@runner_mode} #{options.host}:#{options.port} #{app_file}"
       end
 
       # Configure the mode of the server and the runner class.
-      def configure_server_mode
-        debug_say 'Configuring runner mode'
+      def configure_runner_mode
+        debug_say('Configuring runner mode')
 
         if multi_mode?
-          say 'WARNING: You have provided multiple mode options. Defaulting to socket mode.', :yellow
-          server_socket!
+          say('WARNING: You have provided multiple mode options. Defaulting to socket mode.', :yellow)
+          @runner_mode = :socket
         elsif options.zmq?
-          server_zmq!
+          @runner_mode = :zmq
         elsif options.evented?
-          server_evented!
+          @runner_mode = :evented
         else
           case server_type = ENV["PB_SERVER_TYPE"]
           when nil, /socket/i
-            server_socket!
+            @runner_mode = :socket
           when /zmq/i
-            server_zmq!
+            @runner_mode = :zmq
           when /evented/i
-            server_evented!
+            @runner_mode = :evented
           else
             say "WARNING: You have provided incorrect option 'PB_SERVER_TYPE=#{server_type}'. Defaulting to socket mode.", :yellow
-            server_socket!
+            @runner_mode = :socket
           end
         end
       end
@@ -129,21 +130,36 @@ def configure_server_mode
       # Configure signal traps.
       # TODO add signal handling for hot-reloading the application.
       def configure_traps
-        debug_say 'Configuring traps'
+        debug_say('Configuring traps')
 
         exit_signals = [:INT, :TERM]
         exit_signals << :QUIT unless defined?(JRUBY_VERSION)
 
         exit_signals.each do |signal|
-          debug_say "Registering trap for exit signal #{signal}", :blue
+          debug_say("Registering trap for exit signal #{signal}", :blue)
 
           trap(signal) do
             @exit_requested = true
-            shutdown_server!
+            shutdown_server
           end
         end
       end
 
+      # Create the runner for the configured mode
+      def create_runner
+        debug_say("Creating #{@runner_mode} runner")
+        @runner = case @runner_mode
+                  when :evented
+                    create_evented_runner
+                  when :zmq
+                    create_zmq_runner
+                  when :socket
+                    create_socket_runner
+                  else
+                    say_and_exit("Unknown runner mode: #{@runner_mode}")
+                  end
+      end
+
       # Say something if we're in debug mode.
       def debug_say(message, color = :yellow)
         say(message, color) if options.debug?
@@ -163,21 +179,23 @@ def multi_mode?
       end
 
       # Require the application file given, exiting if the file doesn't exist.
-      def require_application!(app_file)
-        debug_say 'Requiring app file'
+      def require_application(app_file)
+        debug_say('Requiring app file')
         require app_file
       rescue LoadError => e
-        say_and_exit!("Failed to load application file #{app_file}", e)
+        say_and_exit("Failed to load application file #{app_file}", e)
       end
 
       def runner_options
-        opt = options.inject({}){|h,(k,v)|h[k.to_sym]=v;h} # Symbolize keys
+        # Symbolize keys
+        opt = options.inject({}) { |h, (k, v)| h[k.to_sym] = v; h }
+
         opt[:workers_only] = !!ENV['PB_WORKERS_ONLY'] || options.workers_only
 
         opt
       end
 
-      def say_and_exit!(message, exception = nil)
+      def say_and_exit(message, exception = nil)
         message = set_color(message, :red) if ::Protobuf::Logger.file == STDOUT
 
         ::Protobuf::Logger.error { message }
@@ -192,41 +210,38 @@ def say_and_exit!(message, exception = nil)
         exit(1)
       end
 
-      def server_evented!
+      def create_evented_runner
         require 'protobuf/evented'
 
-        @mode = :evented
         @runner = ::Protobuf::Rpc::EventedRunner.new(runner_options)
       end
 
-      def server_socket!
+      def create_socket_runner
         require 'protobuf/socket'
 
-        @mode = :socket
         @runner = ::Protobuf::Rpc::SocketRunner.new(runner_options)
       end
 
-      def server_zmq!
+      def create_zmq_runner
         require 'protobuf/zmq'
 
-        @mode = :zmq
         @runner = ::Protobuf::Rpc::ZmqRunner.new(runner_options)
       end
 
-      def shutdown_server!
+      def shutdown_server
         ::Protobuf::Logger.info { 'RPC Server shutting down...' }
-        @runner.try :stop
+        @runner.try(:stop)
         ::Protobuf::Rpc::ServiceDirectory.instance.stop
         ::Protobuf::Logger.info { 'Shutdown complete' }
       end
 
       # Start the runner and log the relevant options.
-      def start_server!
-        debug_say 'Invoking server start'
+      def start_server
+        debug_say('Running server')
 
         @runner.run do
           ::Protobuf::Logger.info {
-            "pid #{::Process.pid} -- #{@mode} RPC Server listening at #{options.host}:#{options.port}"
+            "pid #{::Process.pid} -- #{@runner_mode} RPC Server listening at #{options.host}:#{options.port}"
           }
         end
       end

From b8c611694205b07bdb131511aa73098d79c15900 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Tue, 11 Jun 2013 13:55:00 -0600
Subject: [PATCH 66/70] Update README

---
 README.md | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 93e94a05..fe0650a8 100644
--- a/README.md
+++ b/README.md
@@ -303,8 +303,7 @@ any other filter calls which would run afterwards, as well as canceling
 invocation of the service method. Note: You must actually return false,
 not just a "falsey" value such as nil.
 
-__After Filters__ – There is no request shortcutting since the after
-filter runs after the request. Duh.
+__After Filters__ – No request shortcutting.
 
 #### Filter options
 
@@ -415,6 +414,29 @@ Many different options can be passed to the `.client` call above
 (such as `:timeout => 600`). See the `lib/protobuf/rpc/client.rb`
 and `lib/protobuf/rpc/service.rb` files for more documentation.
 
+### Dynamic Discovery (ZMQ Only)
+It is possible to setup the RPC server and client in a way that
+allows servers to be dynamically discovered by the client.
+
+#### In the client
+```ruby
+ServiceDirectory.start do |config|
+  config.port = 53000
+end
+```
+
+#### Starting the server
+```
+$ rpc_server -o myserver.com --broadcast-beacons ./config/environment.rb
+```
+
+The client will listen on the specified port for beacons broadcast
+by servers. Each beacon includes a list of services provided by the
+broadcasting server. The client randomly selects a server for the
+desired service each time a request is made.
+
+Check out the {Protobuf::ServiceDirectory} class for more details.
+
 ## 3. RPC Interop
 
 The main reason I wrote this gem was to provide a ruby implementation

From a25ba7aaf8883091c603a340e512cf6fe2a81c20 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Tue, 11 Jun 2013 13:59:19 -0600
Subject: [PATCH 67/70] Add caution about cross-environment beacons

---
 README.md | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index fe0650a8..8b9665d6 100644
--- a/README.md
+++ b/README.md
@@ -435,7 +435,12 @@ by servers. Each beacon includes a list of services provided by the
 broadcasting server. The client randomly selects a server for the
 desired service each time a request is made.
 
-Check out the {Protobuf::ServiceDirectory} class for more details.
+*CAUTION:* When running multiple environments on a single network,
+e.g., qa and staging, be sure that each environment is setup with
+a unique broadcast port; otherwise, clients in one environment will
+make requests to servers in another environment.
+
+Check out {Protobuf::ServiceDirectory} for more details.
 
 ## 3. RPC Interop
 

From 67d9b0e3cedd8a58d69584973d09434312a0c3ed Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Tue, 11 Jun 2013 15:13:20 -0500
Subject: [PATCH 68/70] Update README.md

---
 README.md | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 8b9665d6..00fa0fde 100644
--- a/README.md
+++ b/README.md
@@ -423,11 +423,21 @@ allows servers to be dynamically discovered by the client.
 ServiceDirectory.start do |config|
   config.port = 53000
 end
+
+# If your server also runs this code, it will default to the
+# given port when sending beacons and have its own service
+# directory. You can prevent this code from running on the
+# server if needed:
+unless defined? ::Protobuf::CLI
+  ServiceDirectory.start do |config|
+    config.port = 53000
+  end
+end
 ```
 
 #### Starting the server
 ```
-$ rpc_server -o myserver.com --broadcast-beacons ./config/environment.rb
+$ rpc_server --broadcast-beacons --beacon-port 53000 ...
 ```
 
 The client will listen on the specified port for beacons broadcast
@@ -435,10 +445,10 @@ by servers. Each beacon includes a list of services provided by the
 broadcasting server. The client randomly selects a server for the
 desired service each time a request is made.
 
-*CAUTION:* When running multiple environments on a single network,
+__CAUTION:__ When running multiple environments on a single network,
 e.g., qa and staging, be sure that each environment is setup with
-a unique broadcast port; otherwise, clients in one environment will
-make requests to servers in another environment.
+a unique beacon port; otherwise, clients in one environment _will_
+make requests to servers in the other environment.
 
 Check out {Protobuf::ServiceDirectory} for more details.
 

From 9e15a0bda037a2eecf92526e6f7a8a013f792b21 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 12 Jun 2013 11:42:44 -0600
Subject: [PATCH 69/70] Reconcile with pull-request feedback

---
 lib/protobuf/cli.rb                    |  2 +-
 lib/protobuf/rpc/servers/zmq/broker.rb | 88 ++++++++++++++------------
 lib/protobuf/rpc/servers/zmq/server.rb | 61 +++++++++++++-----
 lib/protobuf/rpc/service_directory.rb  | 38 ++++++-----
 4 files changed, 116 insertions(+), 73 deletions(-)

diff --git a/lib/protobuf/cli.rb b/lib/protobuf/cli.rb
index 3ef067c9..6dc0b909 100644
--- a/lib/protobuf/cli.rb
+++ b/lib/protobuf/cli.rb
@@ -190,7 +190,7 @@ def runner_options
         # Symbolize keys
         opt = options.inject({}) { |h, (k, v)| h[k.to_sym] = v; h }
 
-        opt[:workers_only] = !!ENV['PB_WORKERS_ONLY'] || options.workers_only
+        opt[:workers_only] = (!!ENV['PB_WORKERS_ONLY']) || options.workers_only
 
         opt
       end
diff --git a/lib/protobuf/rpc/servers/zmq/broker.rb b/lib/protobuf/rpc/servers/zmq/broker.rb
index 0e339722..430d12d0 100644
--- a/lib/protobuf/rpc/servers/zmq/broker.rb
+++ b/lib/protobuf/rpc/servers/zmq/broker.rb
@@ -22,36 +22,17 @@ def join
           @thread.try(:join)
         end
 
-        def read_from_backend
-          [].tap do |frames|
-            zmq_error_check(@backend_socket.recv_strings(frames))
-          end
-        end
-
-        def read_from_frontend
-          [].tap do |frames|
-            zmq_error_check(@frontend_socket.recv_strings(frames))
-          end
-        end
-
         def run
-          idle_workers = []
+          @idle_workers = []
 
           catch(:shutdown) do
             while @poller.poll > 0
               @poller.readables.each do |readable|
                 case readable
                 when @frontend_socket
-                  if idle_workers.any?
-                    frames = read_from_frontend
-                    write_to_backend([idle_workers.shift, ""] + frames)
-                  end
+                  process_frontend
                 when @backend_socket
-                  worker, ignore, *frames = read_from_backend
-                  idle_workers << worker
-                  unless frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
-                    write_to_frontend(frames)
-                  end
+                  process_backend
                 when @shutdown_socket
                   throw :shutdown
                 end
@@ -62,20 +43,10 @@ def run
           teardown
         end
 
-        def write_to_backend(frames)
-          zmq_error_check(@backend_socket.send_strings(frames))
-        end
-
-        def write_to_frontend(frames)
-          zmq_error_check(@frontend_socket.send_strings(frames))
-        end
-
         def start
           log_debug { sign_message("starting broker") }
 
-          @thread = Thread.new do
-            self.run
-          end
+          @thread = Thread.new { self.run }
 
           self
         end
@@ -91,13 +62,6 @@ def signal_shutdown
           zmq_error_check(socket.close)
         end
 
-        def teardown
-          @frontend_socket.try(:close)
-          @backend_socket.try(:close)
-          @shutdown_socket.try(:close)
-          @zmq_context.try(:terminate)
-        end
-
         private
 
         def init_backend_socket
@@ -125,6 +89,50 @@ def init_shutdown_socket
         def init_zmq_context
           @zmq_context = ZMQ::Context.new
         end
+
+        def process_backend
+          worker, ignore, *frames = read_from_backend
+
+          @idle_workers << worker
+
+          unless frames == [::Protobuf::Rpc::Zmq::WORKER_READY_MESSAGE]
+            write_to_frontend(frames)
+          end
+        end
+
+        def process_frontend
+          if @idle_workers.any?
+            frames = read_from_frontend
+            write_to_backend([@idle_workers.shift, ""] + frames)
+          end
+        end
+
+        def read_from_backend
+          [].tap do |frames|
+            zmq_error_check(@backend_socket.recv_strings(frames))
+          end
+        end
+
+        def read_from_frontend
+          [].tap do |frames|
+            zmq_error_check(@frontend_socket.recv_strings(frames))
+          end
+        end
+
+        def teardown
+          @frontend_socket.try(:close)
+          @backend_socket.try(:close)
+          @shutdown_socket.try(:close)
+          @zmq_context.try(:terminate)
+        end
+
+        def write_to_backend(frames)
+          zmq_error_check(@backend_socket.send_strings(frames))
+        end
+
+        def write_to_frontend(frames)
+          zmq_error_check(@frontend_socket.send_strings(frames))
+        end
       end
     end
   end
diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 16638847..7ca31458 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -87,6 +87,8 @@ def broadcast_flatline
         end
 
         def broadcast_heartbeat
+          @last_beacon = Time.now.to_i
+
           heartbeat = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new(
             :beacon_type => ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT,
             :server => self.to_proto
@@ -97,6 +99,10 @@ def broadcast_heartbeat
           log_debug { sign_message("sent heartbeat to #{beacon_uri}") }
         end
 
+        def broadcast_heartbeat?
+          Time.now.to_i >= next_beacon && broadcast_beacons?
+        end
+
         def brokerless?
           !!options[:workers_only]
         end
@@ -113,20 +119,45 @@ def frontend_uri
           "tcp://#{frontend_ip}:#{frontend_port}"
         end
 
-        def maintenance_interval
-          [reaping_interval, beacon_interval].min
+        def next_maintenance
+          cycles = [next_reaping]
+          cycles << next_beacon if broadcast_beacons?
+
+          cycles.min
         end
 
         def minimum_timeout
           100
         end
 
+        def next_beacon
+          if @last_beacon.nil?
+            0
+          else
+            Time.now.to_i + beacon_interval
+          end
+        end
+
+        def next_reaping
+          if @last_reaping.nil?
+            0
+          else
+            Time.now.to_i + reaping_interval
+          end
+        end
+
         def reap_dead_workers
+          @last_reaping = Time.now.to_i
+
           @workers.keep_if do |worker|
             worker.alive? or worker.join && false
           end
         end
 
+        def reap_dead_workers?
+          Time.now.to_i >= next_reaping
+        end
+
         def reaping_interval
           5
         end
@@ -195,12 +226,21 @@ def teardown
           @shutdown_socket.try(:close)
           @beacon_socket.try(:close)
           @zmq_context.try(:terminate)
+          @last_reaping = @last_beacon = @timeout = nil
         end
 
         def total_workers
           @total_workers ||= [@options[:threads].to_i, 1].max
         end
 
+        def timeout
+          if @timeout.nil?
+            0
+          else
+            @timeout = [minimum_timeout, 1_000 * next_maintenance].max
+          end
+        end
+
         def to_proto
           @proto ||= ::Protobuf::Rpc::DynamicDiscovery::Server.new(
             :uuid => uuid,
@@ -216,31 +256,18 @@ def uuid
         end
 
         def wait_for_shutdown_signal
-          time = Time.now.to_i
-          timeout = 0
-          next_beacon = 0
-          next_reaping = time + reaping_interval
-          next_cycle = time + maintenance_interval
           poller = ZMQ::Poller.new
           poller.register_readable(@shutdown_socket)
 
           # If the poller returns 1, a shutdown signal has been received.
           # If the poller returns -1, something went wrong.
           while poller.poll(timeout) === 0
-            if (time = Time.now.to_i) >= next_reaping
+            if reap_dead_workers?
               reap_dead_workers
               start_missing_workers
-              next_reaping = time + reaping_interval
-              next_cycle = [next_cycle, next_reaping].min
-            end
-
-            if broadcast_beacons? && time >= next_beacon
-              broadcast_heartbeat
-              next_beacon = time + beacon_interval
-              next_cycle = [next_cycle, next_beacon].min
             end
 
-            timeout = [minimum_timeout, 1000 * (next_cycle - time)].max
+            broadcast_heartbeat if broadcast_heartbeat?
           end
         end
 
diff --git a/lib/protobuf/rpc/service_directory.rb b/lib/protobuf/rpc/service_directory.rb
index 0a36d804..7ef89197 100644
--- a/lib/protobuf/rpc/service_directory.rb
+++ b/lib/protobuf/rpc/service_directory.rb
@@ -172,26 +172,34 @@ def init_socket
         @socket.bind(self.class.address, self.class.port.to_i)
       end
 
+      def process_beacon(beacon)
+        case beacon.beacon_type
+        when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
+          add_listing_for(beacon.server)
+        when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE
+          remove_listing_for(beacon.server)
+        end
+      end
+
       def run
-        begin
-          data, addr = @socket.recvfrom(2048)
-          beacon = ::Protobuf::Rpc::DynamicDiscovery::Beacon.new
+        loop do
+          process_beacon(wait_for_beacon)
+          remove_expired_listings
+        end
+      rescue => e
+        log_debug { sign_message("error: (#{e.class}) #{e.message}") }
+        retry
+      end
+
+      def wait_for_beacon
+        data, addr = @socket.recvfrom(2048)
+
+        ::Protobuf::Rpc::DynamicDiscovery::Beacon.new.tap do |beacon|
           beacon.parse_from_string(data) rescue nil
 
           # Favor the address captured by the socket
           beacon.try(:server).try(:address=, addr[3])
-
-          case beacon.beacon_type
-          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::HEARTBEAT
-            add_listing_for(beacon.server)
-          when ::Protobuf::Rpc::DynamicDiscovery::BeaconType::FLATLINE
-            remove_listing_for(beacon.server)
-          end
-
-          remove_expired_listings
-        rescue => e
-          log_debug { sign_message("error: (#{e.class}) #{e.message}") }
-        end while true
+        end
       end
     end
   end

From 05fd332dfe205eb4c4ca14ea5be425ac481bd9d4 Mon Sep 17 00:00:00 2001
From: Devin Christensen <quixoten@gmail.com>
Date: Wed, 12 Jun 2013 13:40:45 -0600
Subject: [PATCH 70/70] Fix some bugs introduced during refactor

---
 lib/protobuf/rpc/servers/zmq/server.rb | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/protobuf/rpc/servers/zmq/server.rb b/lib/protobuf/rpc/servers/zmq/server.rb
index 7ca31458..39736683 100644
--- a/lib/protobuf/rpc/servers/zmq/server.rb
+++ b/lib/protobuf/rpc/servers/zmq/server.rb
@@ -119,6 +119,10 @@ def frontend_uri
           "tcp://#{frontend_ip}:#{frontend_port}"
         end
 
+        def maintenance_timeout
+          1_000 * (next_maintenance - Time.now.to_i)
+        end
+
         def next_maintenance
           cycles = [next_reaping]
           cycles << next_beacon if broadcast_beacons?
@@ -134,7 +138,7 @@ def next_beacon
           if @last_beacon.nil?
             0
           else
-            Time.now.to_i + beacon_interval
+            @last_beacon + beacon_interval
           end
         end
 
@@ -142,7 +146,7 @@ def next_reaping
           if @last_reaping.nil?
             0
           else
-            Time.now.to_i + reaping_interval
+            @last_reaping + reaping_interval
           end
         end
 
@@ -235,9 +239,9 @@ def total_workers
 
         def timeout
           if @timeout.nil?
-            0
+            @timeout = 0
           else
-            @timeout = [minimum_timeout, 1_000 * next_maintenance].max
+            @timeout = [minimum_timeout, maintenance_timeout].max
           end
         end