Stop reconnect_on_network_failure loop on session#close

reopen session in case automatically_recover is called after close

make explicit close handling more obvious on recovery

lib/march_hare/exceptions.rb

@@ -16,6 +16,12 @@ class NetworkException < Exception
   class ConnectionRefused < NetworkException
   end
 
+  class ConnectionClosedException < Exception
+    def initialize(message='')
+      super("Connection was explicitly closed and cannot be reopened. Create a new Connection instead. #{message}")
+    end
+  end
+
   class ChannelLevelException < Exception
     attr_reader :channel_close
 

lib/march_hare/session.rb

@@ -180,6 +180,7 @@ def initialize(connection_factory, opts = {})
       @shutdown_hooks            = Array.new
       @blocked_connection_hooks  = Array.new
       @connection_recovery_hooks = Array.new
+      @was_explicitly_closed     = false
 
       if @automatically_recover
         self.add_automatic_recovery_hook
@@ -225,8 +226,11 @@ def close
         ch.close
       end
 
+      @was_explicitly_closed = true
       maybe_shut_down_executor
       @connection.close
+    rescue com.rabbitmq.client.AlreadyClosedException
+      @logger.debug("close: connection already closed")
     end
 
     # @return [Boolean] true if connection is open, false otherwise
@@ -318,7 +322,8 @@ def disable_automatic_recovery
     # Begins automatic connection recovery (typically only used internally
     # to recover from network failures)
     def automatically_recover
-      @logger.debug("session: begin automatic connection recovery")
+      raise ConnectionClosedException if @was_explicitly_closed
+      @logger.debug("session: begin automatic connection recovery #{Thread.current.inspect}")
 
       fire_recovery_start_hooks
 
@@ -583,6 +588,7 @@ def converting_rjc_exceptions_to_ruby(&block)
     def reconnecting_on_network_failures(interval_in_ms, &fn)
       @logger.debug("session: reconnecting_on_network_failures")
       begin
+        return if @was_explicitly_closed
         fn.call
       rescue IOError, MarchHare::ConnectionRefused, java.io.IOException, java.util.concurrent.TimeoutException
         java.lang.Thread.sleep(interval_in_ms)

spec/higher_level_api/integration/connection_recovery_spec.rb

@@ -10,7 +10,7 @@
 
   def close_all_connections!
     # wait for stats to refresh, make sure to run bin/ci/before_build.sh as well!
-    sleep 0.7
+    sleep 1.7
     http_client.list_connections.each do |conn_info|
       puts "Closing connection #{conn_info.name}..."
       http_client.close_connection(conn_info.name)

spec/higher_level_api/integration/connection_spec.rb

@@ -41,6 +41,16 @@
     end
   end
 
+  context "when connection manually closed" do
+    it "raises an exception on #automatically_recover" do
+      c = MarchHare.connect(uri: "amqp://127.0.0.1")
+      c.close
+      expect {
+        c.automatically_recover
+      }.to raise_error(MarchHare::ConnectionClosedException)
+    end
+  end
+
   it "handles amqp:// URIs w/o path part" do
     c = MarchHare.connect(uri: "amqp://127.0.0.1")
 
@@ -61,6 +71,7 @@
     end
     c = MarchHare.connect(executor_factory: factory, network_recovery_interval: 0)
     c.close
+    c.instance_variable_set(:@was_explicitly_closed, false)
     c.automatically_recover
     c.close
     expect(calls).to eq(2)
@@ -72,6 +83,7 @@
     expect(c).to be_connected
     c.close
     expect(c).not_to be_connected
+    c.instance_variable_set(:@was_explicitly_closed, false)
     c.automatically_recover
     sleep 0.5
     expect(c).to be_connected
@@ -93,6 +105,7 @@
     expect(c).to be_connected
     c.close
     expect(c).not_to be_connected
+    c.instance_variable_set(:@was_explicitly_closed, false)
     c.automatically_recover
     sleep 0.5
     expect(c).to be_connected
@@ -104,6 +117,7 @@
     expect(c).to be_connected
     c.close
     expect(c).not_to be_connected
+    c.instance_variable_set(:@was_explicitly_closed, false)
     c.automatically_recover
     sleep 0.5
     expect(c).to be_connected