Fix issue with wanfed lan ip conflicts.

Prior to this commit, the connection pools were unaware which datacenter the
connection was associated with. This meant that any time servers with
overlapping LAN IP addresses and node shortnames existed, they would be
incorrectly co-located in the same pool. Whenever this occurred, the servers
would get stuck in an infinite loop of forwarding RPCs to themselves (rather
than the intended remote DC) until they eventually run out of memory.

Most notably, this issue can occur whenever wan federation through mesh
gateways is enabled.

This fix adds extra metadata to specify which DC the connection is associated
with in the pool.

agent/pool/pool.go

@@ -49,6 +49,7 @@ type Conn struct {
 	refCount    int32
 	shouldClose int32
 
+	dc       string
 	nodeName string
 	addr     net.Addr
 	session  muxSession
@@ -234,7 +235,7 @@ func (p *ConnPool) acquire(dc string, nodeName string, addr net.Addr) (*Conn, er
 
 	addrStr := addr.String()
 
-	poolKey := nodeName + ":" + addrStr
+	poolKey := makePoolKey(dc, nodeName, addrStr)
 
 	// Check to see if there's a pooled connection available. This is up
 	// here since it should the vastly more common case than the rest
@@ -493,6 +494,7 @@ func (p *ConnPool) getNewConn(dc string, nodeName string, addr net.Addr) (*Conn,
 	// Wrap the connection
 	c := &Conn{
 		refCount: 1,
+		dc:       dc,
 		nodeName: nodeName,
 		addr:     addr,
 		session:  session,
@@ -514,7 +516,7 @@ func (p *ConnPool) clearConn(conn *Conn) {
 
 	// Clear from the cache
 	addrStr := conn.addr.String()
-	poolKey := conn.nodeName + ":" + addrStr
+	poolKey := makePoolKey(conn.dc, conn.nodeName, addrStr)
 	p.Lock()
 	if c, ok := p.pool[poolKey]; ok && c == conn {
 		delete(p.pool, poolKey)
@@ -716,3 +718,8 @@ func (p *ConnPool) reap() {
 		p.Unlock()
 	}
 }
+
+// makePoolKey generates a unique key for grouping connections together into a pool.
+func makePoolKey(dc, nodeName, addrStr string) string {
+	return dc + ":" + nodeName + ":" + addrStr
+}