From a037ae6c93a5bafb63002e6ce0b7866bb48169e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 6 Apr 2018 16:11:36 +0200 Subject: [PATCH 01/44] p2p: cleanup listening logic, make dial act like ssh -L MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 14 +-- p2p/inbound.go | 70 ++++++++++++++ p2p/outbound.go | 119 ++++++++++++++++++++++++ p2p/p2p.go | 214 +++---------------------------------------- p2p/registry.go | 38 +------- 5 files changed, 213 insertions(+), 242 deletions(-) create mode 100644 p2p/inbound.go create mode 100644 p2p/outbound.go diff --git a/core/commands/p2p.go b/core/commands/p2p.go index ca852fafcd8..15de8744f34 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -105,8 +105,8 @@ var p2pListenerLsCmd = &cmds.Command{ for _, listener := range n.P2P.Listeners.Listeners { output.Listeners = append(output.Listeners, P2PListenerInfoOutput{ - Protocol: listener.Protocol, - Address: listener.Address.String(), + Protocol: listener.Protocol(), + Address: listener.Address(), }) } @@ -267,7 +267,7 @@ can transparently connect to a p2p service. return } - addr, peer, err := ParsePeerParam(req.Arguments()[0]) + _, peer, err := ParsePeerParam(req.Arguments()[0]) if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -284,15 +284,15 @@ can transparently connect to a p2p service. } } - listenerInfo, err := n.P2P.Dial(n.Context(), addr, peer, proto, bindAddr) + listenerInfo, err := n.P2P.Dial(n.Context(), peer, proto, bindAddr) if err != nil { res.SetError(err, cmdkit.ErrNormal) return } output := P2PListenerInfoOutput{ - Protocol: listenerInfo.Protocol, - Address: listenerInfo.Address.String(), + Protocol: listenerInfo.Protocol(), + Address: listenerInfo.Address(), } res.SetOutput(&output) @@ -331,7 +331,7 @@ var p2pListenerCloseCmd = &cmds.Command{ } for _, listener := range n.P2P.Listeners.Listeners { - if !closeAll && listener.Protocol != proto { + if !closeAll && listener.Protocol() != proto { continue } listener.Close() diff --git a/p2p/inbound.go b/p2p/inbound.go new file mode 100644 index 00000000000..4e893192ad6 --- /dev/null +++ b/p2p/inbound.go @@ -0,0 +1,70 @@ +package p2p + +import ( + "context" + + ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" + net "gx/ipfs/QmYj8wdn5sZEHX2XMDWGBvcXJNdzVbaVpHmXvhHBVZepen/go-libp2p-net" + protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" + manet "gx/ipfs/QmcGXGdw9BWDysPJQHxJinjGHha3eEg4vzFETre4woNwcX/go-multiaddr-net" +) + +// inboundListener accepts libp2p streams and proxies them to a manet host +type inboundListener struct { + p2p *P2P + + // Application proto identifier. + proto string + + addr ma.Multiaddr +} + +// NewListener creates new p2p listener +func (p2p *P2P) NewListener(ctx context.Context, proto string, addr ma.Multiaddr) (Listener, error) { + listenerInfo := &inboundListener{ + proto: proto, + } + + p2p.peerHost.SetStreamHandler(protocol.ID(proto), func(remote net.Stream) { + local, err := manet.Dial(addr) + if err != nil { + remote.Reset() + return + } + + stream := StreamInfo{ + Protocol: proto, + + LocalPeer: p2p.identity, + LocalAddr: addr, + + RemotePeer: remote.Conn().RemotePeer(), + RemoteAddr: remote.Conn().RemoteMultiaddr(), + + Local: local, + Remote: remote, + + Registry: &p2p.Streams, + } + + p2p.Streams.Register(&stream) + stream.startStreaming() + }) + + p2p.Listeners.Register(listenerInfo) + + return listenerInfo, nil +} + +func (l *inboundListener) Protocol() string { + return l.proto +} + +func (l *inboundListener) Address() string { + return l.addr.String() +} + +func (l *inboundListener) Close() error { + l.p2p.peerHost.RemoveStreamHandler(protocol.ID(l.proto)) + return l.p2p.Listeners.Deregister(l.proto) +} diff --git a/p2p/outbound.go b/p2p/outbound.go new file mode 100644 index 00000000000..f99881ee3b5 --- /dev/null +++ b/p2p/outbound.go @@ -0,0 +1,119 @@ +package p2p + +import ( + "context" + "errors" + "time" + + ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" + net "gx/ipfs/QmYj8wdn5sZEHX2XMDWGBvcXJNdzVbaVpHmXvhHBVZepen/go-libp2p-net" + protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" + pstore "gx/ipfs/QmZb7hAgQEhW9dBbzBudU39gCeD4zbe6xafD52LUuF4cUN/go-libp2p-peerstore" + manet "gx/ipfs/QmcGXGdw9BWDysPJQHxJinjGHha3eEg4vzFETre4woNwcX/go-multiaddr-net" + peer "gx/ipfs/QmcJukH2sAFjY3HdBKq35WDzWoL3UUu2gt9wdfqZTUyM74/go-libp2p-peer" +) + +// inboundListener accepts libp2p streams and proxies them to a manet host +type outboundListener struct { + ctx context.Context + cancel context.CancelFunc + + p2p *P2P + id peer.ID + + proto string + peer peer.ID + + listener manet.Listener +} + +// Dial creates new P2P stream to a remote listener +func (p2p *P2P) Dial(ctx context.Context, peer peer.ID, proto string, bindAddr ma.Multiaddr) (Listener, error) { + lnet, _, err := manet.DialArgs(bindAddr) + if err != nil { + return nil, err + } + + switch lnet { + case "tcp", "tcp4", "tcp6": + maListener, err := manet.Listen(bindAddr) + if err != nil { + return nil, err + } + + listener := &outboundListener{ + p2p: p2p, + id: p2p.identity, + + proto: proto, + peer: peer, + + listener: maListener, + } + + go listener.acceptConns() + + return listener, nil + default: + return nil, errors.New("unsupported proto: " + lnet) + } +} + +func (l *outboundListener) dial() (net.Stream, error) { + ctx, cancel := context.WithTimeout(l.ctx, time.Second*30) //TODO: configurable? + defer cancel() + + err := l.p2p.peerHost.Connect(ctx, pstore.PeerInfo{ID: l.peer}) + if err != nil { + return nil, err + } + + return l.p2p.peerHost.NewStream(l.ctx, l.peer, protocol.ID(l.proto)) +} + +func (l *outboundListener) acceptConns() { + for { + local, err := l.listener.Accept() + if err != nil { + return + } + + remote, err := l.dial() + if err != nil { + local.Close() + return + } + + stream := StreamInfo{ + Protocol: l.proto, + + LocalPeer: l.id, + LocalAddr: l.listener.Multiaddr(), + + RemotePeer: remote.Conn().RemotePeer(), + RemoteAddr: remote.Conn().RemoteMultiaddr(), + + Local: local, + Remote: remote, + + Registry: &l.p2p.Streams, + } + + l.p2p.Streams.Register(&stream) + stream.startStreaming() + } +} + +func (l *outboundListener) Close() error { + l.listener.Close() + err := l.p2p.Listeners.Deregister(l.proto) + return err +} + +func (l *outboundListener) Protocol() string { + return l.proto +} + +func (l *outboundListener) Address() string { + return "/ipfs/" + l.peer.String() +} diff --git a/p2p/p2p.go b/p2p/p2p.go index d2af892c911..33928ba1c5c 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -1,17 +1,9 @@ package p2p import ( - "context" - "errors" - "time" - - peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" - manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" - ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" - net "gx/ipfs/QmZNJyx9GGCX4GeuHnLB8fxaxMLs4MjTjHokxfQcCd6Nve/go-libp2p-net" - pro "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - pstore "gx/ipfs/Qmda4cPRvSRyox3SqgJN6DfSZGU5TtHufPTp9uXjFj71X6/go-libp2p-peerstore" - p2phost "gx/ipfs/QmeMYW7Nj8jnnEfs9qhm7SxKkoDPUWXu3MsxX6BFwz34tf/go-libp2p-host" + pstore "gx/ipfs/QmZR2XWVVBCtbgBWnQhWk2xcQfaR3W8faQPriAiaaj7rsr/go-libp2p-peerstore" + p2phost "gx/ipfs/Qmb8T6YBBsjYsVGfrihQLfCJveczZnneSBqBKkYEBWDjge/go-libp2p-host" + peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer" ) // P2P structure holds information on currently running streams/listeners @@ -24,6 +16,14 @@ type P2P struct { peerstore pstore.Peerstore } +type Listener interface { + Protocol() string + Address() string + + // Close closes the listener. Does not affect child streams + Close() error +} + // NewP2P creates new P2P struct func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) *P2P { return &P2P{ @@ -33,197 +33,7 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) } } -func (p2p *P2P) newStreamTo(ctx2 context.Context, p peer.ID, protocol string) (net.Stream, error) { - ctx, cancel := context.WithTimeout(ctx2, time.Second*30) //TODO: configurable? - defer cancel() - err := p2p.peerHost.Connect(ctx, pstore.PeerInfo{ID: p}) - if err != nil { - return nil, err - } - return p2p.peerHost.NewStream(ctx2, p, pro.ID(protocol)) -} - -// Dial creates new P2P stream to a remote listener -func (p2p *P2P) Dial(ctx context.Context, addr ma.Multiaddr, peer peer.ID, proto string, bindAddr ma.Multiaddr) (*ListenerInfo, error) { - lnet, _, err := manet.DialArgs(bindAddr) - if err != nil { - return nil, err - } - - listenerInfo := ListenerInfo{ - Identity: p2p.identity, - Protocol: proto, - } - - remote, err := p2p.newStreamTo(ctx, peer, proto) - if err != nil { - return nil, err - } - - switch lnet { - case "tcp", "tcp4", "tcp6": - listener, err := manet.Listen(bindAddr) - if err != nil { - if err2 := remote.Reset(); err2 != nil { - return nil, err2 - } - return nil, err - } - - listenerInfo.Address = listener.Multiaddr() - listenerInfo.Closer = listener - listenerInfo.Running = true - - go p2p.doAccept(&listenerInfo, remote, listener) - - default: - return nil, errors.New("unsupported protocol: " + lnet) - } - - return &listenerInfo, nil -} - -func (p2p *P2P) doAccept(listenerInfo *ListenerInfo, remote net.Stream, listener manet.Listener) { - defer listener.Close() - - local, err := listener.Accept() - if err != nil { - return - } - - stream := StreamInfo{ - Protocol: listenerInfo.Protocol, - - LocalPeer: listenerInfo.Identity, - LocalAddr: listenerInfo.Address, - - RemotePeer: remote.Conn().RemotePeer(), - RemoteAddr: remote.Conn().RemoteMultiaddr(), - - Local: local, - Remote: remote, - - Registry: &p2p.Streams, - } - - p2p.Streams.Register(&stream) - stream.startStreaming() -} - -// Listener wraps stream handler into a listener -type Listener interface { - Accept() (net.Stream, error) - Close() error -} - -// P2PListener holds information on a listener -type P2PListener struct { - peerHost p2phost.Host - conCh chan net.Stream - proto pro.ID - ctx context.Context - cancel func() -} - -// Accept waits for a connection from the listener -func (il *P2PListener) Accept() (net.Stream, error) { - select { - case c := <-il.conCh: - return c, nil - case <-il.ctx.Done(): - return nil, il.ctx.Err() - } -} - -// Close closes the listener and removes stream handler -func (il *P2PListener) Close() error { - il.cancel() - il.peerHost.RemoveStreamHandler(il.proto) - return nil -} - -// Listen creates new P2PListener -func (p2p *P2P) registerStreamHandler(ctx2 context.Context, protocol string) (*P2PListener, error) { - ctx, cancel := context.WithCancel(ctx2) - - list := &P2PListener{ - peerHost: p2p.peerHost, - proto: pro.ID(protocol), - conCh: make(chan net.Stream), - ctx: ctx, - cancel: cancel, - } - - p2p.peerHost.SetStreamHandler(list.proto, func(s net.Stream) { - select { - case list.conCh <- s: - case <-ctx.Done(): - s.Reset() - } - }) - - return list, nil -} - -// NewListener creates new p2p listener -func (p2p *P2P) NewListener(ctx context.Context, proto string, addr ma.Multiaddr) (*ListenerInfo, error) { - listener, err := p2p.registerStreamHandler(ctx, proto) - if err != nil { - return nil, err - } - - listenerInfo := ListenerInfo{ - Identity: p2p.identity, - Protocol: proto, - Address: addr, - Closer: listener, - Running: true, - Registry: &p2p.Listeners, - } - - go p2p.acceptStreams(&listenerInfo, listener) - - p2p.Listeners.Register(&listenerInfo) - - return &listenerInfo, nil -} - -func (p2p *P2P) acceptStreams(listenerInfo *ListenerInfo, listener Listener) { - for listenerInfo.Running { - remote, err := listener.Accept() - if err != nil { - listener.Close() - break - } - - local, err := manet.Dial(listenerInfo.Address) - if err != nil { - remote.Reset() - continue - } - - stream := StreamInfo{ - Protocol: listenerInfo.Protocol, - - LocalPeer: listenerInfo.Identity, - LocalAddr: listenerInfo.Address, - - RemotePeer: remote.Conn().RemotePeer(), - RemoteAddr: remote.Conn().RemoteMultiaddr(), - - Local: local, - Remote: remote, - - Registry: &p2p.Streams, - } - - p2p.Streams.Register(&stream) - stream.startStreaming() - } - p2p.Listeners.Deregister(listenerInfo.Protocol) -} - -// CheckProtoExists checks whether a protocol handler is registered to +// CheckProtoExists checks whether a proto handler is registered to // mux handler func (p2p *P2P) CheckProtoExists(proto string) bool { protos := p2p.peerHost.Mux().Protocols() diff --git a/p2p/registry.go b/p2p/registry.go index 7ea528c5c5a..80a1addefbf 100644 --- a/p2p/registry.go +++ b/p2p/registry.go @@ -10,41 +10,13 @@ import ( net "gx/ipfs/QmZNJyx9GGCX4GeuHnLB8fxaxMLs4MjTjHokxfQcCd6Nve/go-libp2p-net" ) -// ListenerInfo holds information on a p2p listener. -type ListenerInfo struct { - // Application protocol identifier. - Protocol string - - // Node identity - Identity peer.ID - - // Local protocol stream address. - Address ma.Multiaddr - - // Local protocol stream listener. - Closer io.Closer - - // Flag indicating whether we're still accepting incoming connections, or - // whether this application listener has been shutdown. - Running bool - - Registry *ListenerRegistry -} - -// Close closes the listener. Does not affect child streams -func (c *ListenerInfo) Close() error { - c.Closer.Close() - err := c.Registry.Deregister(c.Protocol) - return err -} - -// ListenerRegistry is a collection of local application protocol listeners. +// ListenerRegistry is a collection of local application proto listeners. type ListenerRegistry struct { - Listeners []*ListenerInfo + Listeners []Listener } // Register registers listenerInfo2 in this registry -func (c *ListenerRegistry) Register(listenerInfo *ListenerInfo) { +func (c *ListenerRegistry) Register(listenerInfo Listener) { c.Listeners = append(c.Listeners, listenerInfo) } @@ -52,7 +24,7 @@ func (c *ListenerRegistry) Register(listenerInfo *ListenerInfo) { func (c *ListenerRegistry) Deregister(proto string) error { foundAt := -1 for i, a := range c.Listeners { - if a.Protocol == proto { + if a.Protocol() == proto { foundAt = i break } @@ -120,7 +92,7 @@ func (s *StreamInfo) startStreaming() { }() } -// StreamRegistry is a collection of active incoming and outgoing protocol app streams. +// StreamRegistry is a collection of active incoming and outgoing proto app streams. type StreamRegistry struct { Streams []*StreamInfo From a0ad8cfd5c49d3a2f3a3de1004a8cad65369d315 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 6 Apr 2018 15:59:48 +0200 Subject: [PATCH 02/44] p2p: Optimize registry, move stream stuff around MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 12 ++--- p2p/inbound.go | 5 +- p2p/listener.go | 39 ++++++++++++++ p2p/outbound.go | 6 +-- p2p/p2p.go | 17 ------ p2p/registry.go | 122 ------------------------------------------- p2p/stream.go | 83 +++++++++++++++++++++++++++++ 7 files changed, 134 insertions(+), 150 deletions(-) create mode 100644 p2p/listener.go delete mode 100644 p2p/registry.go create mode 100644 p2p/stream.go diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 15de8744f34..084a64420e7 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -91,7 +91,7 @@ var p2pListenerLsCmd = &cmds.Command{ Tagline: "List active p2p listeners.", }, Options: []cmdkit.Option{ - cmdkit.BoolOption("headers", "v", "Print table headers (HandlerID, Protocol, Local, Remote)."), + cmdkit.BoolOption("headers", "v", "Print table headers (Id, Protocol, Local, Remote)."), }, Run: func(req cmds.Request, res cmds.Response) { @@ -156,7 +156,7 @@ var p2pStreamLsCmd = &cmds.Command{ for _, s := range n.P2P.Streams.Streams { output.Streams = append(output.Streams, P2PStreamInfoOutput{ - HandlerID: strconv.FormatUint(s.HandlerID, 10), + HandlerID: strconv.FormatUint(s.Id, 10), Protocol: s.Protocol, @@ -184,7 +184,7 @@ var p2pStreamLsCmd = &cmds.Command{ w := tabwriter.NewWriter(buf, 1, 2, 1, ' ', 0) for _, stream := range list.Streams { if headers { - fmt.Fprintln(w, "HandlerID\tProtocol\tLocal\tRemote") + fmt.Fprintln(w, "Id\tProtocol\tLocal\tRemote") } fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", stream.HandlerID, stream.Protocol, stream.LocalAddress, stream.RemotePeer) @@ -347,7 +347,7 @@ var p2pStreamCloseCmd = &cmds.Command{ Tagline: "Close active p2p stream.", }, Arguments: []cmdkit.Argument{ - cmdkit.StringArg("HandlerID", false, false, "Stream HandlerID"), + cmdkit.StringArg("Id", false, false, "Stream Id"), }, Options: []cmdkit.Option{ cmdkit.BoolOption("all", "a", "Close all streams."), @@ -366,7 +366,7 @@ var p2pStreamCloseCmd = &cmds.Command{ if !closeAll { if len(req.Arguments()) == 0 { - res.SetError(errors.New("no HandlerID specified"), cmdkit.ErrNormal) + res.SetError(errors.New("no Id specified"), cmdkit.ErrNormal) return } @@ -378,7 +378,7 @@ var p2pStreamCloseCmd = &cmds.Command{ } for _, stream := range n.P2P.Streams.Streams { - if !closeAll && handlerID != stream.HandlerID { + if !closeAll && handlerID != stream.Id { continue } stream.Close() diff --git a/p2p/inbound.go b/p2p/inbound.go index 4e893192ad6..ad54977642b 100644 --- a/p2p/inbound.go +++ b/p2p/inbound.go @@ -32,7 +32,7 @@ func (p2p *P2P) NewListener(ctx context.Context, proto string, addr ma.Multiaddr return } - stream := StreamInfo{ + stream := Stream{ Protocol: proto, LocalPeer: p2p.identity, @@ -66,5 +66,6 @@ func (l *inboundListener) Address() string { func (l *inboundListener) Close() error { l.p2p.peerHost.RemoveStreamHandler(protocol.ID(l.proto)) - return l.p2p.Listeners.Deregister(l.proto) + l.p2p.Listeners.Deregister(l.proto) + return nil } diff --git a/p2p/listener.go b/p2p/listener.go new file mode 100644 index 00000000000..881771c5954 --- /dev/null +++ b/p2p/listener.go @@ -0,0 +1,39 @@ +package p2p + +import ( + pstore "gx/ipfs/QmZb7hAgQEhW9dBbzBudU39gCeD4zbe6xafD52LUuF4cUN/go-libp2p-peerstore" + peer "gx/ipfs/QmcJukH2sAFjY3HdBKq35WDzWoL3UUu2gt9wdfqZTUyM74/go-libp2p-peer" + p2phost "gx/ipfs/QmdHyfNVTZ5VtUx4Xz23z8wtnioSrFQ28XSfpVkdhQBkGA/go-libp2p-host" +) + +type Listener interface { + Protocol() string + Address() string + + // Close closes the listener. Does not affect child streams + Close() error +} + +// NewP2P creates new P2P struct +func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) *P2P { + return &P2P{ + identity: identity, + peerHost: peerHost, + peerstore: peerstore, + } +} + +// ListenerRegistry is a collection of local application proto listeners. +type ListenerRegistry struct { + Listeners map[string]Listener +} + +// Register registers listenerInfo2 in this registry +func (c *ListenerRegistry) Register(listenerInfo Listener) { + c.Listeners[listenerInfo.Protocol()] = listenerInfo +} + +// Deregister removes p2p listener from this registry +func (c *ListenerRegistry) Deregister(proto string) { + delete(c.Listeners, proto) +} diff --git a/p2p/outbound.go b/p2p/outbound.go index f99881ee3b5..13f4733eb52 100644 --- a/p2p/outbound.go +++ b/p2p/outbound.go @@ -84,7 +84,7 @@ func (l *outboundListener) acceptConns() { return } - stream := StreamInfo{ + stream := Stream{ Protocol: l.proto, LocalPeer: l.id, @@ -106,8 +106,8 @@ func (l *outboundListener) acceptConns() { func (l *outboundListener) Close() error { l.listener.Close() - err := l.p2p.Listeners.Deregister(l.proto) - return err + l.p2p.Listeners.Deregister(l.proto) + return nil } func (l *outboundListener) Protocol() string { diff --git a/p2p/p2p.go b/p2p/p2p.go index 33928ba1c5c..72cc3d9251e 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -16,23 +16,6 @@ type P2P struct { peerstore pstore.Peerstore } -type Listener interface { - Protocol() string - Address() string - - // Close closes the listener. Does not affect child streams - Close() error -} - -// NewP2P creates new P2P struct -func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) *P2P { - return &P2P{ - identity: identity, - peerHost: peerHost, - peerstore: peerstore, - } -} - // CheckProtoExists checks whether a proto handler is registered to // mux handler func (p2p *P2P) CheckProtoExists(proto string) bool { diff --git a/p2p/registry.go b/p2p/registry.go deleted file mode 100644 index 80a1addefbf..00000000000 --- a/p2p/registry.go +++ /dev/null @@ -1,122 +0,0 @@ -package p2p - -import ( - "fmt" - "io" - - peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" - manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" - ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" - net "gx/ipfs/QmZNJyx9GGCX4GeuHnLB8fxaxMLs4MjTjHokxfQcCd6Nve/go-libp2p-net" -) - -// ListenerRegistry is a collection of local application proto listeners. -type ListenerRegistry struct { - Listeners []Listener -} - -// Register registers listenerInfo2 in this registry -func (c *ListenerRegistry) Register(listenerInfo Listener) { - c.Listeners = append(c.Listeners, listenerInfo) -} - -// Deregister removes p2p listener from this registry -func (c *ListenerRegistry) Deregister(proto string) error { - foundAt := -1 - for i, a := range c.Listeners { - if a.Protocol() == proto { - foundAt = i - break - } - } - - if foundAt != -1 { - c.Listeners = append(c.Listeners[:foundAt], c.Listeners[foundAt+1:]...) - return nil - } - - return fmt.Errorf("failed to deregister proto %s", proto) -} - -// StreamInfo holds information on active incoming and outgoing p2p streams. -type StreamInfo struct { - HandlerID uint64 - - Protocol string - - LocalPeer peer.ID - LocalAddr ma.Multiaddr - - RemotePeer peer.ID - RemoteAddr ma.Multiaddr - - Local manet.Conn - Remote net.Stream - - Registry *StreamRegistry -} - -// Close closes stream endpoints and deregisters it -func (s *StreamInfo) Close() error { - s.Local.Close() - s.Remote.Close() - s.Registry.Deregister(s.HandlerID) - return nil -} - -// Reset closes stream endpoints and deregisters it -func (s *StreamInfo) Reset() error { - s.Local.Close() - s.Remote.Reset() - s.Registry.Deregister(s.HandlerID) - return nil -} - -func (s *StreamInfo) startStreaming() { - go func() { - _, err := io.Copy(s.Local, s.Remote) - if err != nil { - s.Reset() - } else { - s.Close() - } - }() - - go func() { - _, err := io.Copy(s.Remote, s.Local) - if err != nil { - s.Reset() - } else { - s.Close() - } - }() -} - -// StreamRegistry is a collection of active incoming and outgoing proto app streams. -type StreamRegistry struct { - Streams []*StreamInfo - - nextID uint64 -} - -// Register registers a stream to the registry -func (c *StreamRegistry) Register(streamInfo *StreamInfo) { - streamInfo.HandlerID = c.nextID - c.Streams = append(c.Streams, streamInfo) - c.nextID++ -} - -// Deregister deregisters stream from the registry -func (c *StreamRegistry) Deregister(handlerID uint64) { - foundAt := -1 - for i, s := range c.Streams { - if s.HandlerID == handlerID { - foundAt = i - break - } - } - - if foundAt != -1 { - c.Streams = append(c.Streams[:foundAt], c.Streams[foundAt+1:]...) - } -} diff --git a/p2p/stream.go b/p2p/stream.go new file mode 100644 index 00000000000..0dc936a63a1 --- /dev/null +++ b/p2p/stream.go @@ -0,0 +1,83 @@ +package p2p + +import ( + "io" + + ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" + net "gx/ipfs/QmYj8wdn5sZEHX2XMDWGBvcXJNdzVbaVpHmXvhHBVZepen/go-libp2p-net" + manet "gx/ipfs/QmcGXGdw9BWDysPJQHxJinjGHha3eEg4vzFETre4woNwcX/go-multiaddr-net" + peer "gx/ipfs/QmcJukH2sAFjY3HdBKq35WDzWoL3UUu2gt9wdfqZTUyM74/go-libp2p-peer" +) + +// Stream holds information on active incoming and outgoing p2p streams. +type Stream struct { + Id uint64 + + Protocol string + + LocalPeer peer.ID + LocalAddr ma.Multiaddr + + RemotePeer peer.ID + RemoteAddr ma.Multiaddr + + Local manet.Conn + Remote net.Stream + + Registry *StreamRegistry +} + +// Close closes stream endpoints and deregisters it +func (s *Stream) Close() error { + s.Local.Close() + s.Remote.Close() + s.Registry.Deregister(s.Id) + return nil +} + +// Reset closes stream endpoints and deregisters it +func (s *Stream) Reset() error { + s.Local.Close() + s.Remote.Reset() + s.Registry.Deregister(s.Id) + return nil +} + +func (s *Stream) startStreaming() { + go func() { + _, err := io.Copy(s.Local, s.Remote) + if err != nil { + s.Reset() + } else { + s.Close() + } + }() + + go func() { + _, err := io.Copy(s.Remote, s.Local) + if err != nil { + s.Reset() + } else { + s.Close() + } + }() +} + +// StreamRegistry is a collection of active incoming and outgoing proto app streams. +type StreamRegistry struct { + Streams map[uint64]*Stream + + nextId uint64 +} + +// Register registers a stream to the registry +func (c *StreamRegistry) Register(streamInfo *Stream) { + streamInfo.Id = c.nextId + c.Streams[c.nextId] = streamInfo + c.nextId++ +} + +// Deregister deregisters stream from the registry +func (c *StreamRegistry) Deregister(streamId uint64) { + delete(c.Streams, streamId) +} From 633b04e5da739f89eea1efd5fe6e4b15ef614fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 11 May 2018 15:37:05 +0200 Subject: [PATCH 03/44] p2p: remove redundant listener address check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/outbound.go | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/p2p/outbound.go b/p2p/outbound.go index 13f4733eb52..1d4da9e5377 100644 --- a/p2p/outbound.go +++ b/p2p/outbound.go @@ -2,7 +2,6 @@ package p2p import ( "context" - "errors" "time" ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" @@ -29,34 +28,24 @@ type outboundListener struct { // Dial creates new P2P stream to a remote listener func (p2p *P2P) Dial(ctx context.Context, peer peer.ID, proto string, bindAddr ma.Multiaddr) (Listener, error) { - lnet, _, err := manet.DialArgs(bindAddr) + maListener, err := manet.Listen(bindAddr) if err != nil { return nil, err } - switch lnet { - case "tcp", "tcp4", "tcp6": - maListener, err := manet.Listen(bindAddr) - if err != nil { - return nil, err - } - - listener := &outboundListener{ - p2p: p2p, - id: p2p.identity, + listener := &outboundListener{ + p2p: p2p, + id: p2p.identity, - proto: proto, - peer: peer, + proto: proto, + peer: peer, - listener: maListener, - } + listener: maListener, + } - go listener.acceptConns() + go listener.acceptConns() - return listener, nil - default: - return nil, errors.New("unsupported proto: " + lnet) - } + return listener, nil } func (l *outboundListener) dial() (net.Stream, error) { From 8b7bf26664be15c48580500ba7ef6678481e7410 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 11 May 2018 15:41:54 +0200 Subject: [PATCH 04/44] p2p: add peer address to peerstore if specified MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 084a64420e7..edf90c89173 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -11,8 +11,9 @@ import ( cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" - "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" + pstore "gx/ipfs/QmXauCuJzmzapetmC6W4TuDJLL1yFFrVzSHoWv8YdbmnxH/go-libp2p-peerstore" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" + "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit" ) // P2PListenerInfoOutput is output type of ls command @@ -267,12 +268,16 @@ can transparently connect to a p2p service. return } - _, peer, err := ParsePeerParam(req.Arguments()[0]) + addr, peer, err := ParsePeerParam(req.Arguments()[0]) if err != nil { res.SetError(err, cmdkit.ErrNormal) return } + if addr != nil { + n.Peerstore.AddAddr(peer, addr, pstore.TempAddrTTL) + } + proto := "/p2p/" + req.Arguments()[1] bindAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0") From 132d6fae4a8686f5cdaa8f86b7972f02b17b02e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 22 May 2018 14:21:01 +0200 Subject: [PATCH 05/44] p2p: Only use reset on streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 2 +- p2p/stream.go | 24 ++++-------------------- 2 files changed, 5 insertions(+), 21 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index edf90c89173..917658c8e3b 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -386,7 +386,7 @@ var p2pStreamCloseCmd = &cmds.Command{ if !closeAll && handlerID != stream.Id { continue } - stream.Close() + stream.Reset() if !closeAll { break } diff --git a/p2p/stream.go b/p2p/stream.go index 0dc936a63a1..d9796e28b21 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -27,14 +27,6 @@ type Stream struct { Registry *StreamRegistry } -// Close closes stream endpoints and deregisters it -func (s *Stream) Close() error { - s.Local.Close() - s.Remote.Close() - s.Registry.Deregister(s.Id) - return nil -} - // Reset closes stream endpoints and deregisters it func (s *Stream) Reset() error { s.Local.Close() @@ -45,21 +37,13 @@ func (s *Stream) Reset() error { func (s *Stream) startStreaming() { go func() { - _, err := io.Copy(s.Local, s.Remote) - if err != nil { - s.Reset() - } else { - s.Close() - } + io.Copy(s.Local, s.Remote) + s.Reset() }() go func() { - _, err := io.Copy(s.Remote, s.Local) - if err != nil { - s.Reset() - } else { - s.Close() - } + io.Copy(s.Remote, s.Local) + s.Reset() }() } From cf6ddcbf70cf2a963a90b89281540ad2825935fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 22 May 2018 14:35:41 +0200 Subject: [PATCH 06/44] p2p: fix some stuff after refactor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/inbound.go | 4 ++++ p2p/listener.go | 15 --------------- p2p/outbound.go | 7 ++++--- p2p/p2p.go | 16 ++++++++++++++++ 4 files changed, 24 insertions(+), 18 deletions(-) diff --git a/p2p/inbound.go b/p2p/inbound.go index ad54977642b..03304a37368 100644 --- a/p2p/inbound.go +++ b/p2p/inbound.go @@ -16,13 +16,17 @@ type inboundListener struct { // Application proto identifier. proto string + // Address to proxy the incoming connections to addr ma.Multiaddr } // NewListener creates new p2p listener func (p2p *P2P) NewListener(ctx context.Context, proto string, addr ma.Multiaddr) (Listener, error) { listenerInfo := &inboundListener{ + p2p: p2p, + proto: proto, + addr: addr, } p2p.peerHost.SetStreamHandler(protocol.ID(proto), func(remote net.Stream) { diff --git a/p2p/listener.go b/p2p/listener.go index 881771c5954..dc1fc3009a6 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -1,11 +1,5 @@ package p2p -import ( - pstore "gx/ipfs/QmZb7hAgQEhW9dBbzBudU39gCeD4zbe6xafD52LUuF4cUN/go-libp2p-peerstore" - peer "gx/ipfs/QmcJukH2sAFjY3HdBKq35WDzWoL3UUu2gt9wdfqZTUyM74/go-libp2p-peer" - p2phost "gx/ipfs/QmdHyfNVTZ5VtUx4Xz23z8wtnioSrFQ28XSfpVkdhQBkGA/go-libp2p-host" -) - type Listener interface { Protocol() string Address() string @@ -14,15 +8,6 @@ type Listener interface { Close() error } -// NewP2P creates new P2P struct -func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) *P2P { - return &P2P{ - identity: identity, - peerHost: peerHost, - peerstore: peerstore, - } -} - // ListenerRegistry is a collection of local application proto listeners. type ListenerRegistry struct { Listeners map[string]Listener diff --git a/p2p/outbound.go b/p2p/outbound.go index 1d4da9e5377..b0a5541f206 100644 --- a/p2p/outbound.go +++ b/p2p/outbound.go @@ -12,10 +12,9 @@ import ( peer "gx/ipfs/QmcJukH2sAFjY3HdBKq35WDzWoL3UUu2gt9wdfqZTUyM74/go-libp2p-peer" ) -// inboundListener accepts libp2p streams and proxies them to a manet host +// outboundListener accepts libp2p streams and proxies them to a manet host type outboundListener struct { - ctx context.Context - cancel context.CancelFunc + ctx context.Context p2p *P2P id peer.ID @@ -34,6 +33,8 @@ func (p2p *P2P) Dial(ctx context.Context, peer peer.ID, proto string, bindAddr m } listener := &outboundListener{ + ctx: ctx, + p2p: p2p, id: p2p.identity, diff --git a/p2p/p2p.go b/p2p/p2p.go index 72cc3d9251e..3998dd3b636 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -16,6 +16,22 @@ type P2P struct { peerstore pstore.Peerstore } +// NewP2P creates new P2P struct +func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) *P2P { + return &P2P{ + identity: identity, + peerHost: peerHost, + peerstore: peerstore, + + Listeners: ListenerRegistry{ + Listeners: map[string]Listener{}, + }, + Streams: StreamRegistry{ + Streams: map[uint64]*Stream{}, + }, + } +} + // CheckProtoExists checks whether a proto handler is registered to // mux handler func (p2p *P2P) CheckProtoExists(proto string) bool { From 6a1d709733aac49eb11be82ba92bbd955443c44b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 22 May 2018 17:50:35 +0200 Subject: [PATCH 07/44] p2p: turns out we need half-open streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/outbound.go | 1 + p2p/stream.go | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/p2p/outbound.go b/p2p/outbound.go index b0a5541f206..d5fac26ccc4 100644 --- a/p2p/outbound.go +++ b/p2p/outbound.go @@ -44,6 +44,7 @@ func (p2p *P2P) Dial(ctx context.Context, peer peer.ID, proto string, bindAddr m listener: maListener, } + p2p.Listeners.Register(listener) go listener.acceptConns() return listener, nil diff --git a/p2p/stream.go b/p2p/stream.go index d9796e28b21..ed2bf5f0c9d 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -27,7 +27,15 @@ type Stream struct { Registry *StreamRegistry } -// Reset closes stream endpoints and deregisters it +// Close closes stream endpoints and deregisters it +func (s *Stream) Close() error { + s.Local.Close() + s.Remote.Close() + s.Registry.Deregister(s.Id) + return nil +} + +// Rest closes stream endpoints and deregisters it func (s *Stream) Reset() error { s.Local.Close() s.Remote.Reset() @@ -42,8 +50,12 @@ func (s *Stream) startStreaming() { }() go func() { - io.Copy(s.Remote, s.Local) - s.Reset() + _, err := io.Copy(s.Remote, s.Local) + if err != nil { + s.Reset() + } else { + s.Close() + } }() } From 48d83be05215b8d1886d27541c58c4a43c40d5ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 23 May 2018 15:05:02 +0200 Subject: [PATCH 08/44] p2p: implement forward cmd MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 102 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 1 deletion(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 917658c8e3b..eb64830ec45 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -2,14 +2,17 @@ package commands import ( "bytes" + "context" "errors" "fmt" "io" "strconv" + "strings" "text/tabwriter" cmds "github.com/ipfs/go-ipfs/commands" core "github.com/ipfs/go-ipfs/core" + p2p "github.com/ipfs/go-ipfs/p2p" pstore "gx/ipfs/QmXauCuJzmzapetmC6W4TuDJLL1yFFrVzSHoWv8YdbmnxH/go-libp2p-peerstore" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" @@ -56,9 +59,107 @@ are refined`, Subcommands: map[string]*cmds.Command{ "listener": p2pListenerCmd, "stream": p2pStreamCmd, + "forward": p2pForwardCmd, }, } +var p2pForwardCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Forward connections to or from libp2p services", + ShortDescription: ` +Forward connections to to . Protocol specifies +the libp2p protocol to use. + +To create libp2p service listener, specify '/ipfs' as + +Examples: + ipfs p2p forward myproto /ipfs /ip4/127.0.0.1/tcp/1234 + - Forward connections to 'myproto' libp2p service to 127.0.0.1:1234 + + ipfs p2p forward myproto /ip4/127.0.0.1/tcp/4567 /ipfs/QmPeer + - Forward connections to 127.0.0.1:4567 to 'myproto' service on /ipfs/QmPeer + +`, + }, + Arguments: []cmdkit.Argument{ + cmdkit.StringArg("protocol", true, false, "Protocol identifier."), + cmdkit.StringArg("listen-address", true, false, "Listening endpoint"), + cmdkit.StringArg("target-address", true, false, "Target endpoint."), + }, + Run: func(req cmds.Request, res cmds.Response) { + n, err := getNode(req) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + //TODO: Do we really want/need implicit prefix? + proto := "/p2p/" + req.Arguments()[0] + listen := req.Arguments()[1] + target := req.Arguments()[2] + + if strings.HasPrefix(listen, "/ipfs") { + if listen != "/ipfs" { + res.SetError(errors.New("only '/ipfs' is allowed as libp2p listen address"), cmdkit.ErrNormal) + return + } + + if err := forwardRemote(n.Context(), n.P2P, proto, target); err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + } else { + if err := forwardLocal(n.Context(), n.P2P, n.Peerstore, proto, listen, target); err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + } + res.SetOutput(nil) + }, +} + +// forwardRemote forwards libp2p service connections to a manet address +func forwardRemote(ctx context.Context, p *p2p.P2P, proto string, target string) error { + if strings.HasPrefix(target, "/ipfs") { + return errors.New("cannot forward libp2p service connections to another libp2p service") + } + + addr, err := ma.NewMultiaddr(target) + if err != nil { + return err + } + + // TODO: return some info + _, err = p.NewListener(ctx, proto, addr) + return err +} + +// forwardLocal forwards local connections to a libp2p service +func forwardLocal(ctx context.Context, p *p2p.P2P, ps pstore.Peerstore, proto string, listen string, target string) error { + bindAddr, err := ma.NewMultiaddr(listen) + if err != nil { + return err + } + + addr, peer, err := ParsePeerParam(target) + if err != nil { + return err + } + + if addr != nil { + ps.AddAddr(peer, addr, pstore.TempAddrTTL) + } + + // TODO: return some info + _, err = p.Dial(ctx, peer, proto, bindAddr) + return err +} + +//// +// LEGACY +// +// + // p2pListenerCmd is the 'ipfs p2p listener' command var p2pListenerCmd = &cmds.Command{ Helptext: cmdkit.HelpText{ @@ -95,7 +196,6 @@ var p2pListenerLsCmd = &cmds.Command{ cmdkit.BoolOption("headers", "v", "Print table headers (Id, Protocol, Local, Remote)."), }, Run: func(req cmds.Request, res cmds.Response) { - n, err := getNode(req) if err != nil { res.SetError(err, cmdkit.ErrNormal) From 2487c99a7f8c36c4a783c3632df92be3bdc5eafd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 23 May 2018 16:10:31 +0200 Subject: [PATCH 09/44] p2p: refactor local/remote MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 30 ++++++++++++++++-------------- p2p/listener.go | 3 ++- p2p/{outbound.go => local.go} | 26 +++++++++++++++----------- p2p/{inbound.go => remote.go} | 24 ++++++++++++++---------- 4 files changed, 47 insertions(+), 36 deletions(-) rename p2p/{outbound.go => local.go} (72%) rename p2p/{inbound.go => remote.go} (71%) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index eb64830ec45..375513f8a5f 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -21,8 +21,9 @@ import ( // P2PListenerInfoOutput is output type of ls command type P2PListenerInfoOutput struct { - Protocol string - Address string + Protocol string + ListenAddress string + TargetAddress string } // P2PStreamInfoOutput is output type of streams command @@ -130,7 +131,7 @@ func forwardRemote(ctx context.Context, p *p2p.P2P, proto string, target string) } // TODO: return some info - _, err = p.NewListener(ctx, proto, addr) + _, err = p.ForwardRemote(ctx, proto, addr) return err } @@ -151,7 +152,7 @@ func forwardLocal(ctx context.Context, p *p2p.P2P, ps pstore.Peerstore, proto st } // TODO: return some info - _, err = p.Dial(ctx, peer, proto, bindAddr) + _, err = p.ForwardLocal(ctx, peer, proto, bindAddr) return err } @@ -206,8 +207,9 @@ var p2pListenerLsCmd = &cmds.Command{ for _, listener := range n.P2P.Listeners.Listeners { output.Listeners = append(output.Listeners, P2PListenerInfoOutput{ - Protocol: listener.Protocol(), - Address: listener.Address(), + Protocol: listener.Protocol(), + ListenAddress: listener.ListenAddress(), + TargetAddress: listener.TargetAddress(), }) } @@ -227,10 +229,10 @@ var p2pListenerLsCmd = &cmds.Command{ w := tabwriter.NewWriter(buf, 1, 2, 1, ' ', 0) for _, listener := range list.Listeners { if headers { - fmt.Fprintln(w, "Address\tProtocol") + fmt.Fprintln(w, "Protocol\tListen Address\tTarget Address") } - fmt.Fprintf(w, "%s\t%s\n", listener.Address, listener.Protocol) + fmt.Fprintf(w, "%s\t%s\t%s\n", listener.Protocol, listener.ListenAddress, listener.TargetAddress) } w.Flush() @@ -330,7 +332,7 @@ Note that the connections originate from the ipfs daemon process. return } - _, err = n.P2P.NewListener(n.Context(), proto, addr) + _, err = n.P2P.ForwardRemote(n.Context(), proto, addr) if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -338,8 +340,8 @@ Note that the connections originate from the ipfs daemon process. // Successful response. res.SetOutput(&P2PListenerInfoOutput{ - Protocol: proto, - Address: addr.String(), + Protocol: proto, + TargetAddress: addr.String(), }) }, } @@ -389,15 +391,15 @@ can transparently connect to a p2p service. } } - listenerInfo, err := n.P2P.Dial(n.Context(), peer, proto, bindAddr) + listenerInfo, err := n.P2P.ForwardLocal(n.Context(), peer, proto, bindAddr) if err != nil { res.SetError(err, cmdkit.ErrNormal) return } output := P2PListenerInfoOutput{ - Protocol: listenerInfo.Protocol(), - Address: listenerInfo.Address(), + Protocol: listenerInfo.Protocol(), + ListenAddress: listenerInfo.ListenAddress(), } res.SetOutput(&output) diff --git a/p2p/listener.go b/p2p/listener.go index dc1fc3009a6..b408b8b74df 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -2,7 +2,8 @@ package p2p type Listener interface { Protocol() string - Address() string + ListenAddress() string + TargetAddress() string // Close closes the listener. Does not affect child streams Close() error diff --git a/p2p/outbound.go b/p2p/local.go similarity index 72% rename from p2p/outbound.go rename to p2p/local.go index d5fac26ccc4..bc899850ace 100644 --- a/p2p/outbound.go +++ b/p2p/local.go @@ -12,8 +12,8 @@ import ( peer "gx/ipfs/QmcJukH2sAFjY3HdBKq35WDzWoL3UUu2gt9wdfqZTUyM74/go-libp2p-peer" ) -// outboundListener accepts libp2p streams and proxies them to a manet host -type outboundListener struct { +// localListener manet streams and proxies them to libp2p services +type localListener struct { ctx context.Context p2p *P2P @@ -25,14 +25,14 @@ type outboundListener struct { listener manet.Listener } -// Dial creates new P2P stream to a remote listener -func (p2p *P2P) Dial(ctx context.Context, peer peer.ID, proto string, bindAddr ma.Multiaddr) (Listener, error) { +// ForwardLocal creates new P2P stream to a remote listener +func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto string, bindAddr ma.Multiaddr) (Listener, error) { maListener, err := manet.Listen(bindAddr) if err != nil { return nil, err } - listener := &outboundListener{ + listener := &localListener{ ctx: ctx, p2p: p2p, @@ -50,7 +50,7 @@ func (p2p *P2P) Dial(ctx context.Context, peer peer.ID, proto string, bindAddr m return listener, nil } -func (l *outboundListener) dial() (net.Stream, error) { +func (l *localListener) dial() (net.Stream, error) { ctx, cancel := context.WithTimeout(l.ctx, time.Second*30) //TODO: configurable? defer cancel() @@ -62,7 +62,7 @@ func (l *outboundListener) dial() (net.Stream, error) { return l.p2p.peerHost.NewStream(l.ctx, l.peer, protocol.ID(l.proto)) } -func (l *outboundListener) acceptConns() { +func (l *localListener) acceptConns() { for { local, err := l.listener.Accept() if err != nil { @@ -95,16 +95,20 @@ func (l *outboundListener) acceptConns() { } } -func (l *outboundListener) Close() error { +func (l *localListener) Close() error { l.listener.Close() l.p2p.Listeners.Deregister(l.proto) return nil } -func (l *outboundListener) Protocol() string { +func (l *localListener) Protocol() string { return l.proto } -func (l *outboundListener) Address() string { - return "/ipfs/" + l.peer.String() +func (l *localListener) ListenAddress() string { + return l.listener.Multiaddr().String() +} + +func (l *localListener) TargetAddress() string { + return "/ipfs/" + l.peer.Pretty() } diff --git a/p2p/inbound.go b/p2p/remote.go similarity index 71% rename from p2p/inbound.go rename to p2p/remote.go index 03304a37368..391ebbb0d8a 100644 --- a/p2p/inbound.go +++ b/p2p/remote.go @@ -9,8 +9,8 @@ import ( manet "gx/ipfs/QmcGXGdw9BWDysPJQHxJinjGHha3eEg4vzFETre4woNwcX/go-multiaddr-net" ) -// inboundListener accepts libp2p streams and proxies them to a manet host -type inboundListener struct { +// remoteListener accepts libp2p streams and proxies them to a manet host +type remoteListener struct { p2p *P2P // Application proto identifier. @@ -20,15 +20,17 @@ type inboundListener struct { addr ma.Multiaddr } -// NewListener creates new p2p listener -func (p2p *P2P) NewListener(ctx context.Context, proto string, addr ma.Multiaddr) (Listener, error) { - listenerInfo := &inboundListener{ +// ForwardRemote creates new p2p listener +func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiaddr) (Listener, error) { + listenerInfo := &remoteListener{ p2p: p2p, proto: proto, addr: addr, } + p2p.Listeners.Register(listenerInfo) + p2p.peerHost.SetStreamHandler(protocol.ID(proto), func(remote net.Stream) { local, err := manet.Dial(addr) if err != nil { @@ -55,20 +57,22 @@ func (p2p *P2P) NewListener(ctx context.Context, proto string, addr ma.Multiaddr stream.startStreaming() }) - p2p.Listeners.Register(listenerInfo) - return listenerInfo, nil } -func (l *inboundListener) Protocol() string { +func (l *remoteListener) Protocol() string { return l.proto } -func (l *inboundListener) Address() string { +func (l *remoteListener) ListenAddress() string { + return "/ipfs" +} + +func (l *remoteListener) TargetAddress() string { return l.addr.String() } -func (l *inboundListener) Close() error { +func (l *remoteListener) Close() error { l.p2p.peerHost.RemoveStreamHandler(protocol.ID(l.proto)) l.p2p.Listeners.Deregister(l.proto) return nil From 067154f22fbee2d51776a204ef6b42e675b486c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 23 May 2018 17:13:43 +0200 Subject: [PATCH 10/44] p2p: p2p ls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 62 +++++++++++++++++--------------------------- 1 file changed, 24 insertions(+), 38 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 375513f8a5f..8b0b39023aa 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -58,9 +58,10 @@ are refined`, }, Subcommands: map[string]*cmds.Command{ - "listener": p2pListenerCmd, - "stream": p2pStreamCmd, - "forward": p2pForwardCmd, + "stream": p2pStreamCmd, + + "forward": p2pForwardCmd, + "ls": p2pLsCmd, }, } @@ -156,45 +157,12 @@ func forwardLocal(ctx context.Context, p *p2p.P2P, ps pstore.Peerstore, proto st return err } -//// -// LEGACY -// -// - -// p2pListenerCmd is the 'ipfs p2p listener' command -var p2pListenerCmd = &cmds.Command{ - Helptext: cmdkit.HelpText{ - Tagline: "P2P listener management.", - ShortDescription: "Create and manage listener p2p endpoints", - }, - - Subcommands: map[string]*cmds.Command{ - "ls": p2pListenerLsCmd, - "open": p2pListenerListenCmd, - "close": p2pListenerCloseCmd, - }, -} - -// p2pStreamCmd is the 'ipfs p2p stream' command -var p2pStreamCmd = &cmds.Command{ - Helptext: cmdkit.HelpText{ - Tagline: "P2P stream management.", - ShortDescription: "Create and manage p2p streams", - }, - - Subcommands: map[string]*cmds.Command{ - "ls": p2pStreamLsCmd, - "dial": p2pStreamDialCmd, - "close": p2pStreamCloseCmd, - }, -} - -var p2pListenerLsCmd = &cmds.Command{ +var p2pLsCmd = &cmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "List active p2p listeners.", }, Options: []cmdkit.Option{ - cmdkit.BoolOption("headers", "v", "Print table headers (Id, Protocol, Local, Remote)."), + cmdkit.BoolOption("headers", "v", "Print table headers (Protocol, Listen, Target)."), }, Run: func(req cmds.Request, res cmds.Response) { n, err := getNode(req) @@ -241,6 +209,24 @@ var p2pListenerLsCmd = &cmds.Command{ }, } +/////// +// Listener +// + +// p2pStreamCmd is the 'ipfs p2p stream' command +var p2pStreamCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "P2P stream management.", + ShortDescription: "Create and manage p2p streams", + }, + + Subcommands: map[string]*cmds.Command{ + "ls": p2pStreamLsCmd, + "dial": p2pStreamDialCmd, + "close": p2pStreamCloseCmd, + }, +} + var p2pStreamLsCmd = &cmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "List active p2p streams.", From df6540e014841501ebce0c43735404ffdb0ddfc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 25 May 2018 15:54:39 +0200 Subject: [PATCH 11/44] p2p: rework stream/listener registration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 228 +++++++++++++------------------------------ p2p/listener.go | 26 +++-- p2p/local.go | 21 ++-- p2p/p2p.go | 10 +- p2p/remote.go | 11 +-- p2p/stream.go | 8 +- 6 files changed, 109 insertions(+), 195 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 8b0b39023aa..6046ce453d4 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -30,10 +30,8 @@ type P2PListenerInfoOutput struct { type P2PStreamInfoOutput struct { HandlerID string Protocol string - LocalPeer string - LocalAddress string - RemotePeer string - RemoteAddress string + OriginAddress string + TargetAddress string } // P2PLsOutput is output type of ls command @@ -61,6 +59,7 @@ are refined`, "stream": p2pStreamCmd, "forward": p2pForwardCmd, + "close": p2pCloseCmd, "ls": p2pLsCmd, }, } @@ -209,6 +208,65 @@ var p2pLsCmd = &cmds.Command{ }, } +var p2pCloseCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Stop listening for new connections to forward.", + }, + Options: []cmdkit.Option{ + cmdkit.BoolOption("all", "a", "Close all listeners."), + cmdkit.StringOption("protocol", "p", "Match protocol name"), + cmdkit.StringOption("listen-address", "l", "Match listen address"), + cmdkit.StringOption("target-address", "t", "Match target address"), + }, + Run: func(req cmds.Request, res cmds.Response) { + res.SetOutput(nil) + + n, err := getNode(req) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + closeAll, _, _ := req.Option("all").Bool() + proto, p, _ := req.Option("protocol").String() + listen, l, _ := req.Option("listen-address").String() + target, t, _ := req.Option("target-address").String() + + if !(closeAll || p || l || t) { + res.SetError(errors.New("no connection matching options given"), cmdkit.ErrNormal) + return + } + + if closeAll && (p || l || t) { + res.SetError(errors.New("can't combine --all with other matching options"), cmdkit.ErrNormal) + return + } + + match := func(listener p2p.Listener) bool { + out := true + if p { + out = out && (proto == listener.Protocol()) + } + if l { + out = out && (listen == listener.ListenAddress()) + } + if t { + out = out && (target == listener.TargetAddress()) + } + + out = out || closeAll + return out + } + + for _, listener := range n.P2P.Listeners.Listeners { + if !match(listener) { + continue + } + listener.Close() + } + }, +} + /////// // Listener // @@ -222,7 +280,6 @@ var p2pStreamCmd = &cmds.Command{ Subcommands: map[string]*cmds.Command{ "ls": p2pStreamLsCmd, - "dial": p2pStreamDialCmd, "close": p2pStreamCloseCmd, }, } @@ -249,11 +306,8 @@ var p2pStreamLsCmd = &cmds.Command{ Protocol: s.Protocol, - LocalPeer: s.LocalPeer.Pretty(), - LocalAddress: s.LocalAddr.String(), - - RemotePeer: s.RemotePeer.Pretty(), - RemoteAddress: s.RemoteAddr.String(), + OriginAddress: s.OriginAddr.String(), + TargetAddress: s.TargetAddr.String(), }) } @@ -273,10 +327,10 @@ var p2pStreamLsCmd = &cmds.Command{ w := tabwriter.NewWriter(buf, 1, 2, 1, ' ', 0) for _, stream := range list.Streams { if headers { - fmt.Fprintln(w, "Id\tProtocol\tLocal\tRemote") + fmt.Fprintln(w, "Id\tProtocol\tOrigin\tTarget") } - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", stream.HandlerID, stream.Protocol, stream.LocalAddress, stream.RemotePeer) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", stream.HandlerID, stream.Protocol, stream.OriginAddress, stream.TargetAddress) } w.Flush() @@ -285,156 +339,6 @@ var p2pStreamLsCmd = &cmds.Command{ }, } -var p2pListenerListenCmd = &cmds.Command{ - Helptext: cmdkit.HelpText{ - Tagline: "Forward p2p connections to a network multiaddr.", - ShortDescription: ` -Register a p2p connection handler and forward the connections to a specified -address. - -Note that the connections originate from the ipfs daemon process. - `, - }, - Arguments: []cmdkit.Argument{ - cmdkit.StringArg("Protocol", true, false, "Protocol identifier."), - cmdkit.StringArg("Address", true, false, "Request handling application address."), - }, - Run: func(req cmds.Request, res cmds.Response) { - n, err := getNode(req) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - - proto := "/p2p/" + req.Arguments()[0] - if n.P2P.CheckProtoExists(proto) { - res.SetError(errors.New("protocol handler already registered"), cmdkit.ErrNormal) - return - } - - addr, err := ma.NewMultiaddr(req.Arguments()[1]) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - - _, err = n.P2P.ForwardRemote(n.Context(), proto, addr) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - - // Successful response. - res.SetOutput(&P2PListenerInfoOutput{ - Protocol: proto, - TargetAddress: addr.String(), - }) - }, -} - -var p2pStreamDialCmd = &cmds.Command{ - Helptext: cmdkit.HelpText{ - Tagline: "Dial to a p2p listener.", - - ShortDescription: ` -Establish a new connection to a peer service. - -When a connection is made to a peer service the ipfs daemon will setup one -time TCP listener and return it's bind port, this way a dialing application -can transparently connect to a p2p service. - `, - }, - Arguments: []cmdkit.Argument{ - cmdkit.StringArg("Peer", true, false, "Remote peer to connect to"), - cmdkit.StringArg("Protocol", true, false, "Protocol identifier."), - cmdkit.StringArg("BindAddress", false, false, "Address to listen for connection/s (default: /ip4/127.0.0.1/tcp/0)."), - }, - Run: func(req cmds.Request, res cmds.Response) { - n, err := getNode(req) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - - addr, peer, err := ParsePeerParam(req.Arguments()[0]) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - - if addr != nil { - n.Peerstore.AddAddr(peer, addr, pstore.TempAddrTTL) - } - - proto := "/p2p/" + req.Arguments()[1] - - bindAddr, _ := ma.NewMultiaddr("/ip4/127.0.0.1/tcp/0") - if len(req.Arguments()) == 3 { - bindAddr, err = ma.NewMultiaddr(req.Arguments()[2]) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - } - - listenerInfo, err := n.P2P.ForwardLocal(n.Context(), peer, proto, bindAddr) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - - output := P2PListenerInfoOutput{ - Protocol: listenerInfo.Protocol(), - ListenAddress: listenerInfo.ListenAddress(), - } - - res.SetOutput(&output) - }, -} - -var p2pListenerCloseCmd = &cmds.Command{ - Helptext: cmdkit.HelpText{ - Tagline: "Close active p2p listener.", - }, - Arguments: []cmdkit.Argument{ - cmdkit.StringArg("Protocol", false, false, "P2P listener protocol"), - }, - Options: []cmdkit.Option{ - cmdkit.BoolOption("all", "a", "Close all listeners."), - }, - Run: func(req cmds.Request, res cmds.Response) { - res.SetOutput(nil) - - n, err := getNode(req) - if err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - - closeAll, _, _ := req.Option("all").Bool() - var proto string - - if !closeAll { - if len(req.Arguments()) == 0 { - res.SetError(errors.New("no protocol name specified"), cmdkit.ErrNormal) - return - } - - proto = "/p2p/" + req.Arguments()[0] - } - - for _, listener := range n.P2P.Listeners.Listeners { - if !closeAll && listener.Protocol() != proto { - continue - } - listener.Close() - if !closeAll { - break - } - } - }, -} - var p2pStreamCloseCmd = &cmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "Close active p2p stream.", diff --git a/p2p/listener.go b/p2p/listener.go index b408b8b74df..d51c7a317a1 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -9,17 +9,31 @@ type Listener interface { Close() error } +type listenerKey struct { + proto string + listen string + target string +} + // ListenerRegistry is a collection of local application proto listeners. type ListenerRegistry struct { - Listeners map[string]Listener + Listeners map[listenerKey]Listener } -// Register registers listenerInfo2 in this registry -func (c *ListenerRegistry) Register(listenerInfo Listener) { - c.Listeners[listenerInfo.Protocol()] = listenerInfo +// Register registers listenerInfo in this registry +func (c *ListenerRegistry) Register(l Listener) { + c.Listeners[getListenerKey(l)] = l } // Deregister removes p2p listener from this registry -func (c *ListenerRegistry) Deregister(proto string) { - delete(c.Listeners, proto) +func (c *ListenerRegistry) Deregister(k listenerKey) { + delete(c.Listeners, k) +} + +func getListenerKey(l Listener) listenerKey { + return listenerKey{ + proto: l.Protocol(), + listen: l.ListenAddress(), + target: l.TargetAddress(), + } } diff --git a/p2p/local.go b/p2p/local.go index bc899850ace..acb23ba08c8 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -75,29 +75,32 @@ func (l *localListener) acceptConns() { return } - stream := Stream{ - Protocol: l.proto, + tgt, err := ma.NewMultiaddr(l.TargetAddress()) + if err != nil { + local.Close() + return + } - LocalPeer: l.id, - LocalAddr: l.listener.Multiaddr(), + stream := &Stream{ + Protocol: l.proto, - RemotePeer: remote.Conn().RemotePeer(), - RemoteAddr: remote.Conn().RemoteMultiaddr(), + OriginAddr: local.RemoteMultiaddr(), + TargetAddr: tgt, Local: local, Remote: remote, - Registry: &l.p2p.Streams, + Registry: l.p2p.Streams, } - l.p2p.Streams.Register(&stream) + l.p2p.Streams.Register(stream) stream.startStreaming() } } func (l *localListener) Close() error { l.listener.Close() - l.p2p.Listeners.Deregister(l.proto) + l.p2p.Listeners.Deregister(getListenerKey(l)) return nil } diff --git a/p2p/p2p.go b/p2p/p2p.go index 3998dd3b636..407e658645f 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -8,8 +8,8 @@ import ( // P2P structure holds information on currently running streams/listeners type P2P struct { - Listeners ListenerRegistry - Streams StreamRegistry + Listeners *ListenerRegistry + Streams *StreamRegistry identity peer.ID peerHost p2phost.Host @@ -23,10 +23,10 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) peerHost: peerHost, peerstore: peerstore, - Listeners: ListenerRegistry{ - Listeners: map[string]Listener{}, + Listeners: &ListenerRegistry{ + Listeners: map[listenerKey]Listener{}, }, - Streams: StreamRegistry{ + Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, }, } diff --git a/p2p/remote.go b/p2p/remote.go index 391ebbb0d8a..d466690c505 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -41,16 +41,13 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiad stream := Stream{ Protocol: proto, - LocalPeer: p2p.identity, - LocalAddr: addr, - - RemotePeer: remote.Conn().RemotePeer(), - RemoteAddr: remote.Conn().RemoteMultiaddr(), + OriginAddr: remote.Conn().RemoteMultiaddr(), + TargetAddr: addr, Local: local, Remote: remote, - Registry: &p2p.Streams, + Registry: p2p.Streams, } p2p.Streams.Register(&stream) @@ -74,6 +71,6 @@ func (l *remoteListener) TargetAddress() string { func (l *remoteListener) Close() error { l.p2p.peerHost.RemoveStreamHandler(protocol.ID(l.proto)) - l.p2p.Listeners.Deregister(l.proto) + l.p2p.Listeners.Deregister(getListenerKey(l)) return nil } diff --git a/p2p/stream.go b/p2p/stream.go index ed2bf5f0c9d..f0f0608e1ef 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -6,7 +6,6 @@ import ( ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" net "gx/ipfs/QmYj8wdn5sZEHX2XMDWGBvcXJNdzVbaVpHmXvhHBVZepen/go-libp2p-net" manet "gx/ipfs/QmcGXGdw9BWDysPJQHxJinjGHha3eEg4vzFETre4woNwcX/go-multiaddr-net" - peer "gx/ipfs/QmcJukH2sAFjY3HdBKq35WDzWoL3UUu2gt9wdfqZTUyM74/go-libp2p-peer" ) // Stream holds information on active incoming and outgoing p2p streams. @@ -15,11 +14,8 @@ type Stream struct { Protocol string - LocalPeer peer.ID - LocalAddr ma.Multiaddr - - RemotePeer peer.ID - RemoteAddr ma.Multiaddr + OriginAddr ma.Multiaddr + TargetAddr ma.Multiaddr Local manet.Conn Remote net.Stream From 058edaff589de9f4bf23e05956ae2714088e0d3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 25 May 2018 16:12:50 +0200 Subject: [PATCH 12/44] p2p: make registries thread safer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/listener.go | 19 +++++++++++++++---- p2p/p2p.go | 4 ++++ p2p/remote.go | 4 ++-- p2p/stream.go | 20 ++++++++++++++------ 4 files changed, 35 insertions(+), 12 deletions(-) diff --git a/p2p/listener.go b/p2p/listener.go index d51c7a317a1..972e978805d 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -1,5 +1,9 @@ package p2p +import ( + "sync" +) + type Listener interface { Protocol() string ListenAddress() string @@ -18,16 +22,23 @@ type listenerKey struct { // ListenerRegistry is a collection of local application proto listeners. type ListenerRegistry struct { Listeners map[listenerKey]Listener + lk *sync.Mutex } // Register registers listenerInfo in this registry -func (c *ListenerRegistry) Register(l Listener) { - c.Listeners[getListenerKey(l)] = l +func (r *ListenerRegistry) Register(l Listener) { + r.lk.Lock() + defer r.lk.Unlock() + + r.Listeners[getListenerKey(l)] = l } // Deregister removes p2p listener from this registry -func (c *ListenerRegistry) Deregister(k listenerKey) { - delete(c.Listeners, k) +func (r *ListenerRegistry) Deregister(k listenerKey) { + r.lk.Lock() + defer r.lk.Unlock() + + delete(r.Listeners, k) } func getListenerKey(l Listener) listenerKey { diff --git a/p2p/p2p.go b/p2p/p2p.go index 407e658645f..4bd7bbd4040 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -1,6 +1,8 @@ package p2p import ( + "sync" + pstore "gx/ipfs/QmZR2XWVVBCtbgBWnQhWk2xcQfaR3W8faQPriAiaaj7rsr/go-libp2p-peerstore" p2phost "gx/ipfs/Qmb8T6YBBsjYsVGfrihQLfCJveczZnneSBqBKkYEBWDjge/go-libp2p-host" peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer" @@ -25,9 +27,11 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) Listeners: &ListenerRegistry{ Listeners: map[listenerKey]Listener{}, + lk: &sync.Mutex{}, }, Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, + lk: &sync.Mutex{}, }, } } diff --git a/p2p/remote.go b/p2p/remote.go index d466690c505..3d32f700ecf 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -38,7 +38,7 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiad return } - stream := Stream{ + stream := &Stream{ Protocol: proto, OriginAddr: remote.Conn().RemoteMultiaddr(), @@ -50,7 +50,7 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiad Registry: p2p.Streams, } - p2p.Streams.Register(&stream) + p2p.Streams.Register(stream) stream.startStreaming() }) diff --git a/p2p/stream.go b/p2p/stream.go index f0f0608e1ef..ff5c4a75a28 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -2,6 +2,7 @@ package p2p import ( "io" + "sync" ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" net "gx/ipfs/QmYj8wdn5sZEHX2XMDWGBvcXJNdzVbaVpHmXvhHBVZepen/go-libp2p-net" @@ -58,18 +59,25 @@ func (s *Stream) startStreaming() { // StreamRegistry is a collection of active incoming and outgoing proto app streams. type StreamRegistry struct { Streams map[uint64]*Stream + lk *sync.Mutex nextId uint64 } // Register registers a stream to the registry -func (c *StreamRegistry) Register(streamInfo *Stream) { - streamInfo.Id = c.nextId - c.Streams[c.nextId] = streamInfo - c.nextId++ +func (r *StreamRegistry) Register(streamInfo *Stream) { + r.lk.Lock() + defer r.lk.Unlock() + + streamInfo.Id = r.nextId + r.Streams[r.nextId] = streamInfo + r.nextId++ } // Deregister deregisters stream from the registry -func (c *StreamRegistry) Deregister(streamId uint64) { - delete(c.Streams, streamId) +func (r *StreamRegistry) Deregister(streamId uint64) { + r.lk.Lock() + defer r.lk.Unlock() + + delete(r.Streams, streamId) } From e8ba4fc01af146201c790acfadaed898c072c899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 26 May 2018 15:56:35 +0200 Subject: [PATCH 13/44] p2p: fix sharness tests after refactor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 34 ++++++-- p2p/listener.go | 17 +++- p2p/local.go | 21 +++-- p2p/remote.go | 19 ++++- test/sharness/t0180-p2p.sh | 158 +++++++++++++++++++++++++++---------- 5 files changed, 189 insertions(+), 60 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 6046ce453d4..459923ac262 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -88,7 +88,7 @@ Examples: cmdkit.StringArg("target-address", true, false, "Target endpoint."), }, Run: func(req cmds.Request, res cmds.Response) { - n, err := getNode(req) + n, err := p2pGetNode(req) if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -164,7 +164,7 @@ var p2pLsCmd = &cmds.Command{ cmdkit.BoolOption("headers", "v", "Print table headers (Protocol, Listen, Target)."), }, Run: func(req cmds.Request, res cmds.Response) { - n, err := getNode(req) + n, err := p2pGetNode(req) if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -221,7 +221,7 @@ var p2pCloseCmd = &cmds.Command{ Run: func(req cmds.Request, res cmds.Response) { res.SetOutput(nil) - n, err := getNode(req) + n, err := p2pGetNode(req) if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -244,6 +244,10 @@ var p2pCloseCmd = &cmds.Command{ match := func(listener p2p.Listener) bool { out := true + if p || !strings.HasPrefix(proto, "/p2p/") { + proto = "/p2p/" + proto + } + if p { out = out && (proto == listener.Protocol()) } @@ -258,12 +262,30 @@ var p2pCloseCmd = &cmds.Command{ return out } + var closed int for _, listener := range n.P2P.Listeners.Listeners { if !match(listener) { continue } listener.Close() + closed++ } + res.SetOutput(closed) + }, + Type: int(0), + Marshalers: cmds.MarshalerMap{ + cmds.Text: func(res cmds.Response) (io.Reader, error) { + v, err := unwrapOutput(res.Output()) + if err != nil { + return nil, err + } + + closed := v.(int) + buf := new(bytes.Buffer) + fmt.Fprintf(buf, "Closed %d stream(s)\n", closed) + + return buf, nil + }, }, } @@ -292,7 +314,7 @@ var p2pStreamLsCmd = &cmds.Command{ cmdkit.BoolOption("headers", "v", "Print table headers (HagndlerID, Protocol, Local, Remote)."), }, Run: func(req cmds.Request, res cmds.Response) { - n, err := getNode(req) + n, err := p2pGetNode(req) if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -352,7 +374,7 @@ var p2pStreamCloseCmd = &cmds.Command{ Run: func(req cmds.Request, res cmds.Response) { res.SetOutput(nil) - n, err := getNode(req) + n, err := p2pGetNode(req) if err != nil { res.SetError(err, cmdkit.ErrNormal) return @@ -386,7 +408,7 @@ var p2pStreamCloseCmd = &cmds.Command{ }, } -func getNode(req cmds.Request) (*core.IpfsNode, error) { +func p2pGetNode(req cmds.Request) (*core.IpfsNode, error) { n, err := req.InvocContext().GetNode() if err != nil { return nil, err diff --git a/p2p/listener.go b/p2p/listener.go index 972e978805d..2fca81f3d13 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -2,6 +2,8 @@ package p2p import ( "sync" + + "github.com/pkg/errors" ) type Listener interface { @@ -25,9 +27,22 @@ type ListenerRegistry struct { lk *sync.Mutex } +func (r *ListenerRegistry) Lock(l Listener) error { + r.lk.Lock() + + if _, ok := r.Listeners[getListenerKey(l)]; ok { + r.lk.Unlock() + return errors.New("listener already registered") + } + return nil +} + +func (r *ListenerRegistry) Unlock() { + r.lk.Unlock() +} + // Register registers listenerInfo in this registry func (r *ListenerRegistry) Register(l Listener) { - r.lk.Lock() defer r.lk.Unlock() r.Listeners[getListenerKey(l)] = l diff --git a/p2p/local.go b/p2p/local.go index acb23ba08c8..c34cc703192 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -20,6 +20,7 @@ type localListener struct { id peer.ID proto string + laddr ma.Multiaddr peer peer.ID listener manet.Listener @@ -27,11 +28,6 @@ type localListener struct { // ForwardLocal creates new P2P stream to a remote listener func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto string, bindAddr ma.Multiaddr) (Listener, error) { - maListener, err := manet.Listen(bindAddr) - if err != nil { - return nil, err - } - listener := &localListener{ ctx: ctx, @@ -39,11 +35,22 @@ func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto string, bi id: p2p.identity, proto: proto, + laddr: bindAddr, peer: peer, + } - listener: maListener, + if err := p2p.Listeners.Lock(listener); err != nil { + return nil, err } + maListener, err := manet.Listen(bindAddr) + if err != nil { + p2p.Listeners.Unlock() + return nil, err + } + + listener.listener = maListener + p2p.Listeners.Register(listener) go listener.acceptConns() @@ -109,7 +116,7 @@ func (l *localListener) Protocol() string { } func (l *localListener) ListenAddress() string { - return l.listener.Multiaddr().String() + return l.laddr.String() } func (l *localListener) TargetAddress() string { diff --git a/p2p/remote.go b/p2p/remote.go index 3d32f700ecf..57f4efe9c64 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -22,14 +22,16 @@ type remoteListener struct { // ForwardRemote creates new p2p listener func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiaddr) (Listener, error) { - listenerInfo := &remoteListener{ + listener := &remoteListener{ p2p: p2p, proto: proto, addr: addr, } - p2p.Listeners.Register(listenerInfo) + if err := p2p.Listeners.Lock(listener); err != nil { + return nil, err + } p2p.peerHost.SetStreamHandler(protocol.ID(proto), func(remote net.Stream) { local, err := manet.Dial(addr) @@ -38,10 +40,17 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiad return } + //TODO: review: is there a better way to do this? + peerMa, err := ma.NewMultiaddr("/ipfs/" + remote.Conn().RemotePeer().Pretty()) + if err != nil { + remote.Reset() + return + } + stream := &Stream{ Protocol: proto, - OriginAddr: remote.Conn().RemoteMultiaddr(), + OriginAddr: peerMa, TargetAddr: addr, Local: local, @@ -54,7 +63,9 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiad stream.startStreaming() }) - return listenerInfo, nil + p2p.Listeners.Register(listener) + + return listener, nil } func (l *remoteListener) Protocol() string { diff --git a/test/sharness/t0180-p2p.sh b/test/sharness/t0180-p2p.sh index a11ab35f1d1..71b8b500362 100755 --- a/test/sharness/t0180-p2p.sh +++ b/test/sharness/t0180-p2p.sh @@ -20,11 +20,13 @@ test_expect_success 'peer ids' ' PEERID_0=$(iptb get id 0) && PEERID_1=$(iptb get id 1) ' - -test_expect_success "test ports are closed" ' - (! (netstat -ln | grep "LISTEN" | grep ":10101 ")) && - (! (netstat -ln | grep "LISTEN" | grep ":10102 ")) -' +check_test_ports() { + test_expect_success "test ports are closed" ' + (! (netstat -lnp | grep "LISTEN" | grep ":10101 ")) && + (! (netstat -lnp | grep "LISTEN" | grep ":10102 ")) + ' +} +check_test_ports test_expect_success 'fail without config option being enabled' ' test_must_fail ipfsi 0 p2p stream ls @@ -36,51 +38,105 @@ test_expect_success "enable filestore config setting" ' ' test_expect_success 'start p2p listener' ' - ipfsi 0 p2p listener open p2p-test /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log + ipfsi 0 p2p forward p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log ' -test_expect_success 'Test server to client communications' ' - ma-pipe-unidir --listen --pidFile=listener.pid send /ip4/127.0.0.1/tcp/10101 < test0.bin & +# Server to client communications - test_wait_for_file 30 100ms listener.pid && - kill -0 $(cat listener.pid) && +spawn_sending_server() { + test_expect_success 'S->C Spawn sending server' ' + ma-pipe-unidir --listen --pidFile=listener.pid send /ip4/127.0.0.1/tcp/10101 < test0.bin & - ipfsi 1 p2p stream dial $PEERID_0 p2p-test /ip4/127.0.0.1/tcp/10102 2>&1 > dialer-stdouterr.log && - ma-pipe-unidir recv /ip4/127.0.0.1/tcp/10102 > client.out && - test ! -f listener.pid + test_wait_for_file 30 100ms listener.pid && + kill -0 $(cat listener.pid) + ' +} + +test_server_to_client() { + test_expect_success 'S->C Connect and receive data' ' + ma-pipe-unidir recv /ip4/127.0.0.1/tcp/10102 > client.out + ' + + test_expect_success 'S->C Ensure server finished' ' + test ! -f listener.pid + ' + + test_expect_success 'S->C Output looks good' ' + test_cmp client.out test0.bin + ' +} + +spawn_sending_server + +test_expect_success 'S->C Setup client side' ' + ipfsi 1 p2p forward p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/${PEERID_0} 2>&1 > dialer-stdouterr.log ' -test_expect_success 'Test client to server communications' ' +test_server_to_client + +test_expect_success 'S->C Connect with dead server' ' + ma-pipe-unidir recv /ip4/127.0.0.1/tcp/10102 > client.out +' + +test_expect_success 'S->C Output is empty' ' + test_must_be_empty client.out +' + +spawn_sending_server + +test_server_to_client + +test_expect_success 'S->C Close local listener' ' + ipfsi 1 p2p close -p p2p-test +' + +check_test_ports + +# Client to server communications + +test_expect_success 'C->S Spawn receiving server' ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 > server.out & test_wait_for_file 30 100ms listener.pid && - kill -0 $(cat listener.pid) && + kill -0 $(cat listener.pid) +' + +test_expect_success 'C->S Setup client side' ' + ipfsi 1 p2p forward p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/${PEERID_0} 2>&1 > dialer-stdouterr.log +' + +test_expect_success 'C->S Connect and receive data' ' + ma-pipe-unidir send /ip4/127.0.0.1/tcp/10102 < test1.bin +' - ipfsi 1 p2p stream dial $PEERID_0 p2p-test /ip4/127.0.0.1/tcp/10102 2>&1 > dialer-stdouterr.log && - ma-pipe-unidir send /ip4/127.0.0.1/tcp/10102 < test1.bin && +test_expect_success 'C->S Ensure server finished' ' go-sleep 250ms && test ! -f listener.pid ' -test_expect_success 'server to client output looks good' ' - test_cmp client.out test0.bin +test_expect_success 'C->S Output looks good' ' + test_cmp server.out test1.bin ' -test_expect_success 'client to server output looks good' ' - test_cmp server.out test1.bin +test_expect_success 'C->S Close local listener' ' + ipfsi 1 p2p close -p p2p-test ' -test_expect_success "'ipfs listener p2p ls' succeeds" ' - echo "/ip4/127.0.0.1/tcp/10101 /p2p/p2p-test" > expected && - ipfsi 0 p2p listener ls > actual +check_test_ports + +# Listing streams + +test_expect_success "'ipfs p2p ls' succeeds" ' + echo "/p2p/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101" > expected && + ipfsi 0 p2p ls > actual ' -test_expect_success "'ipfs p2p listener ls' output looks good" ' +test_expect_success "'ipfs p2p ls' output looks good" ' test_cmp expected actual ' test_expect_success "Cannot re-register app handler" ' - (! ipfsi 0 p2p listener open p2p-test /ip4/127.0.0.1/tcp/10101) + test_must_fail ipfsi 0 p2p forward p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 ' test_expect_success "'ipfs p2p stream ls' output is empty" ' @@ -91,7 +147,7 @@ test_expect_success "'ipfs p2p stream ls' output is empty" ' test_expect_success "Setup: Idle stream" ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 & - ipfsi 1 p2p stream dial $PEERID_0 p2p-test /ip4/127.0.0.1/tcp/10102 2>&1 > dialer-stdouterr.log && + ipfsi 1 p2p forward p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && ma-pipe-unidir --pidFile=client.pid recv /ip4/127.0.0.1/tcp/10102 & test_wait_for_file 30 100ms listener.pid && @@ -100,7 +156,7 @@ test_expect_success "Setup: Idle stream" ' ' test_expect_success "'ipfs p2p stream ls' succeeds" ' - echo "2 /p2p/p2p-test /ip4/127.0.0.1/tcp/10101 $PEERID_1" > expected + echo "3 /p2p/p2p-test /ipfs/$PEERID_1 /ip4/127.0.0.1/tcp/10101" > expected ipfsi 0 p2p stream ls > actual ' @@ -109,23 +165,31 @@ test_expect_success "'ipfs p2p stream ls' output looks good" ' ' test_expect_success "'ipfs p2p stream close' closes stream" ' - ipfsi 0 p2p stream close 2 && + ipfsi 0 p2p stream close 3 && ipfsi 0 p2p stream ls > actual && [ ! -f listener.pid ] && [ ! -f client.pid ] && test_must_be_empty actual ' -test_expect_success "'ipfs p2p listener close' closes app handler" ' - ipfsi 0 p2p listener close p2p-test && - ipfsi 0 p2p listener ls > actual && +test_expect_success "'ipfs p2p close' closes remote handler" ' + ipfsi 0 p2p close -p p2p-test && + ipfsi 0 p2p ls > actual && test_must_be_empty actual ' +test_expect_success "'ipfs p2p close' closes local handler" ' + ipfsi 1 p2p close -p p2p-test && + ipfsi 1 p2p ls > actual && + test_must_be_empty actual +' + +check_test_ports + test_expect_success "Setup: Idle stream(2)" ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 & - ipfsi 0 p2p listener open p2p-test2 /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && - ipfsi 1 p2p stream dial $PEERID_0 p2p-test2 /ip4/127.0.0.1/tcp/10102 2>&1 > dialer-stdouterr.log && + ipfsi 0 p2p forward p2p-test2 /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && + ipfsi 1 p2p forward p2p-test2 /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && ma-pipe-unidir --pidFile=client.pid recv /ip4/127.0.0.1/tcp/10102 & test_wait_for_file 30 100ms listener.pid && @@ -134,14 +198,20 @@ test_expect_success "Setup: Idle stream(2)" ' ' test_expect_success "'ipfs p2p stream ls' succeeds(2)" ' - echo "3 /p2p/p2p-test2 /ip4/127.0.0.1/tcp/10101 $PEERID_1" > expected + echo "4 /p2p/p2p-test2 /ipfs/$PEERID_1 /ip4/127.0.0.1/tcp/10101" > expected ipfsi 0 p2p stream ls > actual test_cmp expected actual ' -test_expect_success "'ipfs p2p listener close -a' closes app handlers" ' - ipfsi 0 p2p listener close -a && - ipfsi 0 p2p listener ls > actual && +test_expect_success "'ipfs p2p close -a' closes remote app handlers" ' + ipfsi 0 p2p close -a && + ipfsi 0 p2p ls > actual && + test_must_be_empty actual +' + +test_expect_success "'ipfs p2p close -a' closes local app handlers" ' + ipfsi 1 p2p close -a && + ipfsi 1 p2p ls > actual && test_must_be_empty actual ' @@ -152,13 +222,17 @@ test_expect_success "'ipfs p2p stream close -a' closes streams" ' test_must_be_empty actual ' -test_expect_success "'ipfs p2p listener close' closes app numeric handlers" ' - ipfsi 0 p2p listener open 1234 /ip4/127.0.0.1/tcp/10101 && - ipfsi 0 p2p listener close 1234 && - ipfsi 0 p2p listener ls > actual && +check_test_ports + +test_expect_success "'ipfs p2p close' closes app numeric handlers" ' + ipfsi 0 p2p forward 1234 /ipfs /ip4/127.0.0.1/tcp/10101 && + ipfsi 0 p2p close -p 1234 && + ipfsi 0 p2p ls > actual && test_must_be_empty actual ' +check_test_ports + test_expect_success 'stop iptb' ' iptb stop ' From 0198693e696a2057696f6586c01a8375fc115b64 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 28 Mar 2018 23:51:47 -0700 Subject: [PATCH 14/44] improve the documentation for the p2p feature License: MIT Signed-off-by: Steven Allen --- docs/experimental-features.md | 58 +++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 13 deletions(-) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index ad52b2f4e28..3a0084be97b 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -250,36 +250,68 @@ configured, the daemon will fail to start. --- ## ipfs p2p -Allows to tunnel TCP connections through Libp2p streams + +Allows tunneling of TCP connections through Libp2p streams. If you've ever used +port forwarding with SSH (the `-L` option in openssh), this feature is quite +similar. ### State + Experimental ### In Version + master, 0.4.10 ### How to enable -P2P command needs to be enabled in config -`ipfs config --json Experimental.Libp2pStreamMounting true` +The `p2p` command needs to be enabled in config: + +```sh +> ipfs config --json Experimental.Libp2pStreamMounting true +``` ### How to use -Basic usage: +First, pick a protocol name for your application. Think of the protocol name as +a port number, just significantly more user-friendly. In this example, we're +going to use `/p2p/kickass/1.0`. + +**Setup:** + +1. A "server" node with peer ID `$SERVER_ID` +2. A "client" node. + +**On the "server" node:** + +First, start your application and have it listen on `$APP_PORT`. + +Then, configure the p2p listener by running: + +```sh +> ipfs p2p listener open /p2p/kickass/1.0 /ip4/127.0.0.1/tcp/$APP_PORT +``` + +This will configure IPFS to forward all incoming `/p2p/kickass/1.0` streams to +`127.0.0.1:$APP_PORT` (opening a new connection to `127.0.0.1:$APP_PORT` per +incoming stream. + +**On the "client" node:** + +First, configure the p2p dialer to forward all inbound connections on +`127.0.0.1:SOME_PORT` to the listener behind `/p2p/kickass/1.0` on the server +node. + +```sh +> ipfs p2p stream dial $SERVER_ID /p2p/kickass/1.0 /ip4/127.0.0.1/tcp/$SOME_PORT +``` -- Open a listener on one node (node A) -`ipfs p2p listener open p2p-test /ip4/127.0.0.1/tcp/10101` -- Where `/ip4/127.0.0.1/tcp/10101` put address of application you want to pass - p2p connections to -- On the other node, connect to the listener on node A -`ipfs p2p stream dial $NODE_A_PEERID p2p-test /ip4/127.0.0.1/tcp/10102` -- Node B is now listening for a connection on TCP at 127.0.0.1:10102, connect - your application there to complete the connection +Next, have your application open a connection to `127.0.0.1:$SOME_PORT`. This connection will be forwarded to the service running on `127.0.0.1:$APP_PORT` on the remote machine. ### Road to being a real feature - [ ] Needs more people to use and report on how well it works / fits use cases - [ ] More documentation -- [ ] Support other protocols +- [ ] Support other protocols (e.g, unix domain sockets) --- From e4db7f7a481d80b8112ca14b9382d93dd1873fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 26 May 2018 16:15:39 +0200 Subject: [PATCH 15/44] p2p: update docs after refactor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- docs/experimental-features.md | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 3a0084be97b..b59f3e0e6e3 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -275,7 +275,7 @@ The `p2p` command needs to be enabled in config: First, pick a protocol name for your application. Think of the protocol name as a port number, just significantly more user-friendly. In this example, we're -going to use `/p2p/kickass/1.0`. +going to use `/kickass/1.0`. **Setup:** @@ -289,7 +289,7 @@ First, start your application and have it listen on `$APP_PORT`. Then, configure the p2p listener by running: ```sh -> ipfs p2p listener open /p2p/kickass/1.0 /ip4/127.0.0.1/tcp/$APP_PORT +> ipfs p2p forward /kickass/1.0 /ipfs /ip4/127.0.0.1/tcp/$APP_PORT ``` This will configure IPFS to forward all incoming `/p2p/kickass/1.0` streams to @@ -303,10 +303,28 @@ First, configure the p2p dialer to forward all inbound connections on node. ```sh -> ipfs p2p stream dial $SERVER_ID /p2p/kickass/1.0 /ip4/127.0.0.1/tcp/$SOME_PORT +> ipfs p2p forward /kickass/1.0 /ip4/127.0.0.1/tcp/$SOME_PORT /ipfs/$SERVER_ID ``` -Next, have your application open a connection to `127.0.0.1:$SOME_PORT`. This connection will be forwarded to the service running on `127.0.0.1:$APP_PORT` on the remote machine. +Next, have your application open a connection to `127.0.0.1:$SOME_PORT`. This +connection will be forwarded to the service running on `127.0.0.1:$APP_PORT` on +the remote machine. You can test it with netcat: + +**On "server" node:** +```sh +> nc -v -l -p $APP_PORT +``` + +**On "client" node:** +```sh +> nc -v 127.0.0.1 $SOME_PORT +``` + +You should now see that a connection has been established and be able to +exchange messages between netcat instances. + +(note that depending on your netcat version you may need to drop the `-v` flag) + ### Road to being a real feature - [ ] Needs more people to use and report on how well it works / fits use cases From 0465079f3565c7586d83f1048cc155c234022434 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 26 May 2018 16:18:09 +0200 Subject: [PATCH 16/44] p2p: fix ci MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/commands_test.go | 8 +++----- p2p/listener.go | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index a365f3e1bed..4944a32d393 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -161,13 +161,11 @@ func TestCommands(t *testing.T) { "/object/put", "/object/stat", "/p2p", - "/p2p/listener", - "/p2p/listener/close", - "/p2p/listener/ls", - "/p2p/listener/open", + "/p2p/close", + "/p2p/forward", + "/p2p/ls", "/p2p/stream", "/p2p/stream/close", - "/p2p/stream/dial", "/p2p/stream/ls", "/pin", "/pin/add", diff --git a/p2p/listener.go b/p2p/listener.go index 2fca81f3d13..1f6e16a3371 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -1,9 +1,8 @@ package p2p import ( + "errors" "sync" - - "github.com/pkg/errors" ) type Listener interface { From 3e0184bdad2dd4a3bc678701553d5f23bfa48f3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 26 May 2018 16:32:18 +0200 Subject: [PATCH 17/44] p2p: fix codeclimate warnings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 12 ++++++------ p2p/listener.go | 5 +++-- p2p/local.go | 4 ++-- p2p/remote.go | 2 +- p2p/stream.go | 20 ++++++++++---------- 5 files changed, 22 insertions(+), 21 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 459923ac262..bcfae4fc4d9 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -322,9 +322,9 @@ var p2pStreamLsCmd = &cmds.Command{ output := &P2PStreamsOutput{} - for _, s := range n.P2P.Streams.Streams { + for id, s := range n.P2P.Streams.Streams { output.Streams = append(output.Streams, P2PStreamInfoOutput{ - HandlerID: strconv.FormatUint(s.Id, 10), + HandlerID: strconv.FormatUint(id, 10), Protocol: s.Protocol, @@ -366,7 +366,7 @@ var p2pStreamCloseCmd = &cmds.Command{ Tagline: "Close active p2p stream.", }, Arguments: []cmdkit.Argument{ - cmdkit.StringArg("Id", false, false, "Stream Id"), + cmdkit.StringArg("id", false, false, "Stream identifier"), }, Options: []cmdkit.Option{ cmdkit.BoolOption("all", "a", "Close all streams."), @@ -385,7 +385,7 @@ var p2pStreamCloseCmd = &cmds.Command{ if !closeAll { if len(req.Arguments()) == 0 { - res.SetError(errors.New("no Id specified"), cmdkit.ErrNormal) + res.SetError(errors.New("no id specified"), cmdkit.ErrNormal) return } @@ -396,8 +396,8 @@ var p2pStreamCloseCmd = &cmds.Command{ } } - for _, stream := range n.P2P.Streams.Streams { - if !closeAll && handlerID != stream.Id { + for id, stream := range n.P2P.Streams.Streams { + if !closeAll && handlerID != id { continue } stream.Reset() diff --git a/p2p/listener.go b/p2p/listener.go index 1f6e16a3371..37ac5d8562f 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -5,6 +5,7 @@ import ( "sync" ) +// Listener listens for connections and proxies them to a target type Listener interface { Protocol() string ListenAddress() string @@ -26,7 +27,7 @@ type ListenerRegistry struct { lk *sync.Mutex } -func (r *ListenerRegistry) Lock(l Listener) error { +func (r *ListenerRegistry) lock(l Listener) error { r.lk.Lock() if _, ok := r.Listeners[getListenerKey(l)]; ok { @@ -36,7 +37,7 @@ func (r *ListenerRegistry) Lock(l Listener) error { return nil } -func (r *ListenerRegistry) Unlock() { +func (r *ListenerRegistry) unlock() { r.lk.Unlock() } diff --git a/p2p/local.go b/p2p/local.go index c34cc703192..8e2897ecd18 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -39,13 +39,13 @@ func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto string, bi peer: peer, } - if err := p2p.Listeners.Lock(listener); err != nil { + if err := p2p.Listeners.lock(listener); err != nil { return nil, err } maListener, err := manet.Listen(bindAddr) if err != nil { - p2p.Listeners.Unlock() + p2p.Listeners.unlock() return nil, err } diff --git a/p2p/remote.go b/p2p/remote.go index 57f4efe9c64..8daab8e341d 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -29,7 +29,7 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiad addr: addr, } - if err := p2p.Listeners.Lock(listener); err != nil { + if err := p2p.Listeners.lock(listener); err != nil { return nil, err } diff --git a/p2p/stream.go b/p2p/stream.go index ff5c4a75a28..5858f7fa5c1 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -11,7 +11,7 @@ import ( // Stream holds information on active incoming and outgoing p2p streams. type Stream struct { - Id uint64 + id uint64 Protocol string @@ -28,15 +28,15 @@ type Stream struct { func (s *Stream) Close() error { s.Local.Close() s.Remote.Close() - s.Registry.Deregister(s.Id) + s.Registry.Deregister(s.id) return nil } -// Rest closes stream endpoints and deregisters it +// Reset closes stream endpoints and deregisters it func (s *Stream) Reset() error { s.Local.Close() s.Remote.Reset() - s.Registry.Deregister(s.Id) + s.Registry.Deregister(s.id) return nil } @@ -61,7 +61,7 @@ type StreamRegistry struct { Streams map[uint64]*Stream lk *sync.Mutex - nextId uint64 + nextID uint64 } // Register registers a stream to the registry @@ -69,15 +69,15 @@ func (r *StreamRegistry) Register(streamInfo *Stream) { r.lk.Lock() defer r.lk.Unlock() - streamInfo.Id = r.nextId - r.Streams[r.nextId] = streamInfo - r.nextId++ + streamInfo.id = r.nextID + r.Streams[r.nextID] = streamInfo + r.nextID++ } // Deregister deregisters stream from the registry -func (r *StreamRegistry) Deregister(streamId uint64) { +func (r *StreamRegistry) Deregister(streamID uint64) { r.lk.Lock() defer r.lk.Unlock() - delete(r.Streams, streamId) + delete(r.Streams, streamID) } From 830ed487033f1d4b52d0a1305a7153a72a37529d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 2 Jun 2018 17:10:48 +0200 Subject: [PATCH 18/44] p2p: refactor first review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 18 ++++++++++-------- docs/experimental-features.md | 19 +++++++++++-------- p2p/listener.go | 2 +- p2p/local.go | 8 ++++---- p2p/p2p.go | 4 ++-- p2p/remote.go | 10 +++++----- p2p/stream.go | 5 +++-- 7 files changed, 36 insertions(+), 30 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index bcfae4fc4d9..aa7def1b3ac 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -68,10 +68,12 @@ var p2pForwardCmd = &cmds.Command{ Helptext: cmdkit.HelpText{ Tagline: "Forward connections to or from libp2p services", ShortDescription: ` -Forward connections to to . Protocol specifies -the libp2p protocol to use. +Forward connections made to to . -To create libp2p service listener, specify '/ipfs' as + specifies the libp2p protocol name to use for libp2p +connections and/or handlers. + +To create a libp2p service listener, specify '/ipfs' as Examples: ipfs p2p forward myproto /ipfs /ip4/127.0.0.1/tcp/1234 @@ -83,8 +85,8 @@ Examples: `, }, Arguments: []cmdkit.Argument{ - cmdkit.StringArg("protocol", true, false, "Protocol identifier."), - cmdkit.StringArg("listen-address", true, false, "Listening endpoint"), + cmdkit.StringArg("protocol", true, false, "Protocol name."), + cmdkit.StringArg("listen-address", true, false, "Listening endpoint."), cmdkit.StringArg("target-address", true, false, "Target endpoint."), }, Run: func(req cmds.Request, res cmds.Response) { @@ -311,7 +313,7 @@ var p2pStreamLsCmd = &cmds.Command{ Tagline: "List active p2p streams.", }, Options: []cmdkit.Option{ - cmdkit.BoolOption("headers", "v", "Print table headers (HagndlerID, Protocol, Local, Remote)."), + cmdkit.BoolOption("headers", "v", "Print table headers (ID, Protocol, Local, Remote)."), }, Run: func(req cmds.Request, res cmds.Response) { n, err := p2pGetNode(req) @@ -326,7 +328,7 @@ var p2pStreamLsCmd = &cmds.Command{ output.Streams = append(output.Streams, P2PStreamInfoOutput{ HandlerID: strconv.FormatUint(id, 10), - Protocol: s.Protocol, + Protocol: string(s.Protocol), OriginAddress: s.OriginAddr.String(), TargetAddress: s.TargetAddr.String(), @@ -349,7 +351,7 @@ var p2pStreamLsCmd = &cmds.Command{ w := tabwriter.NewWriter(buf, 1, 2, 1, ' ', 0) for _, stream := range list.Streams { if headers { - fmt.Fprintln(w, "Id\tProtocol\tOrigin\tTarget") + fmt.Fprintln(w, "ID\tProtocol\tOrigin\tTarget") } fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", stream.HandlerID, stream.Protocol, stream.OriginAddress, stream.TargetAddress) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index b59f3e0e6e3..cff1039db7b 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -282,9 +282,12 @@ going to use `/kickass/1.0`. 1. A "server" node with peer ID `$SERVER_ID` 2. A "client" node. -**On the "server" node:** +**Netcat example:** -First, start your application and have it listen on `$APP_PORT`. +***On the "server" node:*** + +First, start your application and have it listen for TCP connections on +port `$APP_PORT`. Then, configure the p2p listener by running: @@ -296,11 +299,11 @@ This will configure IPFS to forward all incoming `/p2p/kickass/1.0` streams to `127.0.0.1:$APP_PORT` (opening a new connection to `127.0.0.1:$APP_PORT` per incoming stream. -**On the "client" node:** +***On the "client" node:*** -First, configure the p2p dialer to forward all inbound connections on -`127.0.0.1:SOME_PORT` to the listener behind `/p2p/kickass/1.0` on the server -node. +First, configure the client p2p dialer, so that it forwards all inbound +connections on `127.0.0.1:SOME_PORT` to the server node listening +on `/p2p/kickass/1.0`. ```sh > ipfs p2p forward /kickass/1.0 /ip4/127.0.0.1/tcp/$SOME_PORT /ipfs/$SERVER_ID @@ -310,12 +313,12 @@ Next, have your application open a connection to `127.0.0.1:$SOME_PORT`. This connection will be forwarded to the service running on `127.0.0.1:$APP_PORT` on the remote machine. You can test it with netcat: -**On "server" node:** +***On "server" node:*** ```sh > nc -v -l -p $APP_PORT ``` -**On "client" node:** +***On "client" node:*** ```sh > nc -v 127.0.0.1 $SOME_PORT ``` diff --git a/p2p/listener.go b/p2p/listener.go index 37ac5d8562f..5ba37a6645d 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -24,7 +24,7 @@ type listenerKey struct { // ListenerRegistry is a collection of local application proto listeners. type ListenerRegistry struct { Listeners map[listenerKey]Listener - lk *sync.Mutex + lk sync.Mutex } func (r *ListenerRegistry) lock(l Listener) error { diff --git a/p2p/local.go b/p2p/local.go index 8e2897ecd18..01e3526a11c 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -19,7 +19,7 @@ type localListener struct { p2p *P2P id peer.ID - proto string + proto protocol.ID laddr ma.Multiaddr peer peer.ID @@ -34,7 +34,7 @@ func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto string, bi p2p: p2p, id: p2p.identity, - proto: proto, + proto: protocol.ID(proto), laddr: bindAddr, peer: peer, } @@ -66,7 +66,7 @@ func (l *localListener) dial() (net.Stream, error) { return nil, err } - return l.p2p.peerHost.NewStream(l.ctx, l.peer, protocol.ID(l.proto)) + return l.p2p.peerHost.NewStream(l.ctx, l.peer, l.proto) } func (l *localListener) acceptConns() { @@ -112,7 +112,7 @@ func (l *localListener) Close() error { } func (l *localListener) Protocol() string { - return l.proto + return string(l.proto) } func (l *localListener) ListenAddress() string { diff --git a/p2p/p2p.go b/p2p/p2p.go index 4bd7bbd4040..16a66b58a01 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -27,11 +27,11 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) Listeners: &ListenerRegistry{ Listeners: map[listenerKey]Listener{}, - lk: &sync.Mutex{}, + lk: sync.Mutex{}, }, Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, - lk: &sync.Mutex{}, + lk: sync.Mutex{}, }, } } diff --git a/p2p/remote.go b/p2p/remote.go index 8daab8e341d..0a0de7735dd 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -14,7 +14,7 @@ type remoteListener struct { p2p *P2P // Application proto identifier. - proto string + proto protocol.ID // Address to proxy the incoming connections to addr ma.Multiaddr @@ -25,7 +25,7 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiad listener := &remoteListener{ p2p: p2p, - proto: proto, + proto: protocol.ID(proto), addr: addr, } @@ -33,7 +33,7 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiad return nil, err } - p2p.peerHost.SetStreamHandler(protocol.ID(proto), func(remote net.Stream) { + p2p.peerHost.SetStreamHandler(listener.proto, func(remote net.Stream) { local, err := manet.Dial(addr) if err != nil { remote.Reset() @@ -48,7 +48,7 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiad } stream := &Stream{ - Protocol: proto, + Protocol: listener.proto, OriginAddr: peerMa, TargetAddr: addr, @@ -69,7 +69,7 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiad } func (l *remoteListener) Protocol() string { - return l.proto + return string(l.proto) } func (l *remoteListener) ListenAddress() string { diff --git a/p2p/stream.go b/p2p/stream.go index 5858f7fa5c1..af60fea0c2a 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -6,6 +6,7 @@ import ( ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" net "gx/ipfs/QmYj8wdn5sZEHX2XMDWGBvcXJNdzVbaVpHmXvhHBVZepen/go-libp2p-net" + "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" manet "gx/ipfs/QmcGXGdw9BWDysPJQHxJinjGHha3eEg4vzFETre4woNwcX/go-multiaddr-net" ) @@ -13,7 +14,7 @@ import ( type Stream struct { id uint64 - Protocol string + Protocol protocol.ID OriginAddr ma.Multiaddr TargetAddr ma.Multiaddr @@ -59,7 +60,7 @@ func (s *Stream) startStreaming() { // StreamRegistry is a collection of active incoming and outgoing proto app streams. type StreamRegistry struct { Streams map[uint64]*Stream - lk *sync.Mutex + lk sync.Mutex nextID uint64 } From 9ba8bc56ee99065a6ebf1890fb072219e1dc5767 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 2 Jun 2018 17:21:13 +0200 Subject: [PATCH 19/44] p2p: additional example MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- docs/experimental-features.md | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index cff1039db7b..b8cb655d592 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -273,17 +273,17 @@ The `p2p` command needs to be enabled in config: ### How to use +**Netcat example:** + First, pick a protocol name for your application. Think of the protocol name as a port number, just significantly more user-friendly. In this example, we're going to use `/kickass/1.0`. -**Setup:** +***Setup:*** 1. A "server" node with peer ID `$SERVER_ID` 2. A "client" node. -**Netcat example:** - ***On the "server" node:*** First, start your application and have it listen for TCP connections on @@ -328,6 +328,31 @@ exchange messages between netcat instances. (note that depending on your netcat version you may need to drop the `-v` flag) +**SSH example** + +**Setup:** + +1. A "server" node with peer ID `$SERVER_ID` and running ssh server on the + default port. +2. A "client" node. + +_you can get `$SERVER_ID` by running `ipfs id -f "\n"`_ + +***First, on the "server" node:*** + +```sh +ipfs p2p forward ssh /ipfs /ip4/127.0.0.1/tcp/22 +``` + +***Then, on "client" node:*** + +```sh +ipfs p2p forward ssh /ip4/127.0.0.1/tcp/2222 /ipfs/$SERVER_ID +``` + +You should now be able to connect to your ssh server through a libp2p connection +with `ssh [user]@127.0.0.1 -p 2222`. + ### Road to being a real feature - [ ] Needs more people to use and report on how well it works / fits use cases From df6decd31cddf8ca7e94f389f77e2c6469a923f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 2 Jun 2018 19:41:16 +0200 Subject: [PATCH 20/44] p2p: don't automatically prefix proto with /p2p MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 11 ++++++----- docs/experimental-features.md | 10 +++++----- test/sharness/t0180-p2p.sh | 32 +++++++++++++++++++------------- 3 files changed, 30 insertions(+), 23 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index aa7def1b3ac..95c6cc02d37 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -96,11 +96,15 @@ Examples: return } - //TODO: Do we really want/need implicit prefix? - proto := "/p2p/" + req.Arguments()[0] + proto := req.Arguments()[0] listen := req.Arguments()[1] target := req.Arguments()[2] + if !strings.HasPrefix(proto, "/p2p/") { + res.SetError(errors.New("protocol name must be within '/p2p/' namespace"), cmdkit.ErrNormal) + return + } + if strings.HasPrefix(listen, "/ipfs") { if listen != "/ipfs" { res.SetError(errors.New("only '/ipfs' is allowed as libp2p listen address"), cmdkit.ErrNormal) @@ -246,9 +250,6 @@ var p2pCloseCmd = &cmds.Command{ match := func(listener p2p.Listener) bool { out := true - if p || !strings.HasPrefix(proto, "/p2p/") { - proto = "/p2p/" + proto - } if p { out = out && (proto == listener.Protocol()) diff --git a/docs/experimental-features.md b/docs/experimental-features.md index b8cb655d592..400e6fe7b71 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -277,7 +277,7 @@ The `p2p` command needs to be enabled in config: First, pick a protocol name for your application. Think of the protocol name as a port number, just significantly more user-friendly. In this example, we're -going to use `/kickass/1.0`. +going to use `/p2p/kickass/1.0`. ***Setup:*** @@ -292,7 +292,7 @@ port `$APP_PORT`. Then, configure the p2p listener by running: ```sh -> ipfs p2p forward /kickass/1.0 /ipfs /ip4/127.0.0.1/tcp/$APP_PORT +> ipfs p2p forward /p2p/kickass/1.0 /ipfs /ip4/127.0.0.1/tcp/$APP_PORT ``` This will configure IPFS to forward all incoming `/p2p/kickass/1.0` streams to @@ -306,7 +306,7 @@ connections on `127.0.0.1:SOME_PORT` to the server node listening on `/p2p/kickass/1.0`. ```sh -> ipfs p2p forward /kickass/1.0 /ip4/127.0.0.1/tcp/$SOME_PORT /ipfs/$SERVER_ID +> ipfs p2p forward /p2p/kickass/1.0 /ip4/127.0.0.1/tcp/$SOME_PORT /ipfs/$SERVER_ID ``` Next, have your application open a connection to `127.0.0.1:$SOME_PORT`. This @@ -341,13 +341,13 @@ _you can get `$SERVER_ID` by running `ipfs id -f "\n"`_ ***First, on the "server" node:*** ```sh -ipfs p2p forward ssh /ipfs /ip4/127.0.0.1/tcp/22 +ipfs p2p forward /p2p/ssh /ipfs /ip4/127.0.0.1/tcp/22 ``` ***Then, on "client" node:*** ```sh -ipfs p2p forward ssh /ip4/127.0.0.1/tcp/2222 /ipfs/$SERVER_ID +ipfs p2p forward /p2p/ssh /ip4/127.0.0.1/tcp/2222 /ipfs/$SERVER_ID ``` You should now be able to connect to your ssh server through a libp2p connection diff --git a/test/sharness/t0180-p2p.sh b/test/sharness/t0180-p2p.sh index 71b8b500362..940ce109211 100755 --- a/test/sharness/t0180-p2p.sh +++ b/test/sharness/t0180-p2p.sh @@ -38,7 +38,7 @@ test_expect_success "enable filestore config setting" ' ' test_expect_success 'start p2p listener' ' - ipfsi 0 p2p forward p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log + ipfsi 0 p2p forward /p2p/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log ' # Server to client communications @@ -69,7 +69,7 @@ test_server_to_client() { spawn_sending_server test_expect_success 'S->C Setup client side' ' - ipfsi 1 p2p forward p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/${PEERID_0} 2>&1 > dialer-stdouterr.log + ipfsi 1 p2p forward /p2p/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/${PEERID_0} 2>&1 > dialer-stdouterr.log ' test_server_to_client @@ -87,7 +87,7 @@ spawn_sending_server test_server_to_client test_expect_success 'S->C Close local listener' ' - ipfsi 1 p2p close -p p2p-test + ipfsi 1 p2p close -p /p2p/p2p-test ' check_test_ports @@ -102,7 +102,7 @@ test_expect_success 'C->S Spawn receiving server' ' ' test_expect_success 'C->S Setup client side' ' - ipfsi 1 p2p forward p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/${PEERID_0} 2>&1 > dialer-stdouterr.log + ipfsi 1 p2p forward /p2p/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/${PEERID_0} 2>&1 > dialer-stdouterr.log ' test_expect_success 'C->S Connect and receive data' ' @@ -119,7 +119,7 @@ test_expect_success 'C->S Output looks good' ' ' test_expect_success 'C->S Close local listener' ' - ipfsi 1 p2p close -p p2p-test + ipfsi 1 p2p close -p /p2p/p2p-test ' check_test_ports @@ -136,7 +136,7 @@ test_expect_success "'ipfs p2p ls' output looks good" ' ' test_expect_success "Cannot re-register app handler" ' - test_must_fail ipfsi 0 p2p forward p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 + test_must_fail ipfsi 0 p2p forward /p2p/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 ' test_expect_success "'ipfs p2p stream ls' output is empty" ' @@ -147,7 +147,7 @@ test_expect_success "'ipfs p2p stream ls' output is empty" ' test_expect_success "Setup: Idle stream" ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 & - ipfsi 1 p2p forward p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && + ipfsi 1 p2p forward /p2p/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && ma-pipe-unidir --pidFile=client.pid recv /ip4/127.0.0.1/tcp/10102 & test_wait_for_file 30 100ms listener.pid && @@ -172,13 +172,13 @@ test_expect_success "'ipfs p2p stream close' closes stream" ' ' test_expect_success "'ipfs p2p close' closes remote handler" ' - ipfsi 0 p2p close -p p2p-test && + ipfsi 0 p2p close -p /p2p/p2p-test && ipfsi 0 p2p ls > actual && test_must_be_empty actual ' test_expect_success "'ipfs p2p close' closes local handler" ' - ipfsi 1 p2p close -p p2p-test && + ipfsi 1 p2p close -p /p2p/p2p-test && ipfsi 1 p2p ls > actual && test_must_be_empty actual ' @@ -188,8 +188,8 @@ check_test_ports test_expect_success "Setup: Idle stream(2)" ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 & - ipfsi 0 p2p forward p2p-test2 /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && - ipfsi 1 p2p forward p2p-test2 /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && + ipfsi 0 p2p forward /p2p/p2p-test2 /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && + ipfsi 1 p2p forward /p2p/p2p-test2 /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && ma-pipe-unidir --pidFile=client.pid recv /ip4/127.0.0.1/tcp/10102 & test_wait_for_file 30 100ms listener.pid && @@ -225,12 +225,18 @@ test_expect_success "'ipfs p2p stream close -a' closes streams" ' check_test_ports test_expect_success "'ipfs p2p close' closes app numeric handlers" ' - ipfsi 0 p2p forward 1234 /ipfs /ip4/127.0.0.1/tcp/10101 && - ipfsi 0 p2p close -p 1234 && + ipfsi 0 p2p forward /p2p/1234 /ipfs /ip4/127.0.0.1/tcp/10101 && + ipfsi 0 p2p close -p /p2p/1234 && ipfsi 0 p2p ls > actual && test_must_be_empty actual ' +test_expect_success "non /p2p/ scoped protocols are not allowed" ' + test_must_fail ipfsi 0 p2p forward /its/not/a/p2p/path /ipfs /ip4/127.0.0.1/tcp/10101 2> actual && + echo "Error: protocol name must be within '"'"'/p2p/'"'"' namespace" > expected + test_cmp expected actual +' + check_test_ports test_expect_success 'stop iptb' ' From df0e566e98dee7420bdff9f8ca9b3ec6dc9b446a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 14 Jun 2018 16:52:20 +0200 Subject: [PATCH 21/44] p2p: update deps after rebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/local.go | 10 +++++----- p2p/remote.go | 6 +++--- p2p/stream.go | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/p2p/local.go b/p2p/local.go index 01e3526a11c..e07dbed7521 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -4,12 +4,12 @@ import ( "context" "time" - ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" - net "gx/ipfs/QmYj8wdn5sZEHX2XMDWGBvcXJNdzVbaVpHmXvhHBVZepen/go-libp2p-net" + manet "gx/ipfs/QmNqRnejxJxjRroz7buhrjfU8i3yNBLa81hFtmf2pXEffN/go-multiaddr-net" + ma "gx/ipfs/QmUxSEGbv2nmYNnfXi7839wwQqTN3kwQeUxe8dTjZWZs7J/go-multiaddr" + peer "gx/ipfs/QmVf8hTAsLLFtn4WPCRNdnaF2Eag2qTBS6uR8AiHPZARXy/go-libp2p-peer" + net "gx/ipfs/QmXdgNhVEgjLxjUoMs5ViQL7pboAt3Y7V7eGHRiE4qrmTE/go-libp2p-net" protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - pstore "gx/ipfs/QmZb7hAgQEhW9dBbzBudU39gCeD4zbe6xafD52LUuF4cUN/go-libp2p-peerstore" - manet "gx/ipfs/QmcGXGdw9BWDysPJQHxJinjGHha3eEg4vzFETre4woNwcX/go-multiaddr-net" - peer "gx/ipfs/QmcJukH2sAFjY3HdBKq35WDzWoL3UUu2gt9wdfqZTUyM74/go-libp2p-peer" + pstore "gx/ipfs/QmZhsmorLpD9kmQ4ynbAu4vbKv2goMUnXazwGA4gnWHDjB/go-libp2p-peerstore" ) // localListener manet streams and proxies them to libp2p services diff --git a/p2p/remote.go b/p2p/remote.go index 0a0de7735dd..dc78612d9fa 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -3,10 +3,10 @@ package p2p import ( "context" - ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" - net "gx/ipfs/QmYj8wdn5sZEHX2XMDWGBvcXJNdzVbaVpHmXvhHBVZepen/go-libp2p-net" + manet "gx/ipfs/QmNqRnejxJxjRroz7buhrjfU8i3yNBLa81hFtmf2pXEffN/go-multiaddr-net" + ma "gx/ipfs/QmUxSEGbv2nmYNnfXi7839wwQqTN3kwQeUxe8dTjZWZs7J/go-multiaddr" + net "gx/ipfs/QmXdgNhVEgjLxjUoMs5ViQL7pboAt3Y7V7eGHRiE4qrmTE/go-libp2p-net" protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - manet "gx/ipfs/QmcGXGdw9BWDysPJQHxJinjGHha3eEg4vzFETre4woNwcX/go-multiaddr-net" ) // remoteListener accepts libp2p streams and proxies them to a manet host diff --git a/p2p/stream.go b/p2p/stream.go index af60fea0c2a..c6e6e398a89 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -4,10 +4,10 @@ import ( "io" "sync" - ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr" - net "gx/ipfs/QmYj8wdn5sZEHX2XMDWGBvcXJNdzVbaVpHmXvhHBVZepen/go-libp2p-net" + manet "gx/ipfs/QmNqRnejxJxjRroz7buhrjfU8i3yNBLa81hFtmf2pXEffN/go-multiaddr-net" + ma "gx/ipfs/QmUxSEGbv2nmYNnfXi7839wwQqTN3kwQeUxe8dTjZWZs7J/go-multiaddr" + net "gx/ipfs/QmXdgNhVEgjLxjUoMs5ViQL7pboAt3Y7V7eGHRiE4qrmTE/go-libp2p-net" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - manet "gx/ipfs/QmcGXGdw9BWDysPJQHxJinjGHha3eEg4vzFETre4woNwcX/go-multiaddr-net" ) // Stream holds information on active incoming and outgoing p2p streams. From c19102816f6328c247a691f43a65e197d17334a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Jun 2018 01:31:27 +0200 Subject: [PATCH 22/44] p2p: change the required prefix to /x/ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 14 +++++++------ docs/experimental-features.md | 14 ++++++------- test/sharness/t0180-p2p.sh | 38 +++++++++++++++++------------------ 3 files changed, 34 insertions(+), 32 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 95c6cc02d37..daa904e43b9 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -19,6 +19,8 @@ import ( "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit" ) +const P2PProtoPrefix = "/x/" + // P2PListenerInfoOutput is output type of ls command type P2PListenerInfoOutput struct { Protocol string @@ -71,16 +73,16 @@ var p2pForwardCmd = &cmds.Command{ Forward connections made to to . specifies the libp2p protocol name to use for libp2p -connections and/or handlers. +connections and/or handlers. It must be prefixed with '` + P2PProtoPrefix + `'. To create a libp2p service listener, specify '/ipfs' as Examples: - ipfs p2p forward myproto /ipfs /ip4/127.0.0.1/tcp/1234 + ipfs p2p forward ` + P2PProtoPrefix + `myproto /ipfs /ip4/127.0.0.1/tcp/1234 - Forward connections to 'myproto' libp2p service to 127.0.0.1:1234 - ipfs p2p forward myproto /ip4/127.0.0.1/tcp/4567 /ipfs/QmPeer - - Forward connections to 127.0.0.1:4567 to 'myproto' service on /ipfs/QmPeer + ipfs p2p forward ` + P2PProtoPrefix + `myproto /ip4/127.0.0.1/tcp/4567 /ipfs/QmPeer + - Forward connections to 127.0.0.1:4567 to '` + P2PProtoPrefix + `myproto' service on /ipfs/QmPeer `, }, @@ -100,8 +102,8 @@ Examples: listen := req.Arguments()[1] target := req.Arguments()[2] - if !strings.HasPrefix(proto, "/p2p/") { - res.SetError(errors.New("protocol name must be within '/p2p/' namespace"), cmdkit.ErrNormal) + if !strings.HasPrefix(proto, P2PProtoPrefix) { + res.SetError(errors.New("protocol name must be within '" + P2PProtoPrefix + "' namespace"), cmdkit.ErrNormal) return } diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 400e6fe7b71..d714e624223 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -277,7 +277,7 @@ The `p2p` command needs to be enabled in config: First, pick a protocol name for your application. Think of the protocol name as a port number, just significantly more user-friendly. In this example, we're -going to use `/p2p/kickass/1.0`. +going to use `/x/kickass/1.0`. ***Setup:*** @@ -292,10 +292,10 @@ port `$APP_PORT`. Then, configure the p2p listener by running: ```sh -> ipfs p2p forward /p2p/kickass/1.0 /ipfs /ip4/127.0.0.1/tcp/$APP_PORT +> ipfs p2p forward /x/kickass/1.0 /ipfs /ip4/127.0.0.1/tcp/$APP_PORT ``` -This will configure IPFS to forward all incoming `/p2p/kickass/1.0` streams to +This will configure IPFS to forward all incoming `/x/kickass/1.0` streams to `127.0.0.1:$APP_PORT` (opening a new connection to `127.0.0.1:$APP_PORT` per incoming stream. @@ -303,10 +303,10 @@ incoming stream. First, configure the client p2p dialer, so that it forwards all inbound connections on `127.0.0.1:SOME_PORT` to the server node listening -on `/p2p/kickass/1.0`. +on `/x/kickass/1.0`. ```sh -> ipfs p2p forward /p2p/kickass/1.0 /ip4/127.0.0.1/tcp/$SOME_PORT /ipfs/$SERVER_ID +> ipfs p2p forward /x/kickass/1.0 /ip4/127.0.0.1/tcp/$SOME_PORT /ipfs/$SERVER_ID ``` Next, have your application open a connection to `127.0.0.1:$SOME_PORT`. This @@ -341,13 +341,13 @@ _you can get `$SERVER_ID` by running `ipfs id -f "\n"`_ ***First, on the "server" node:*** ```sh -ipfs p2p forward /p2p/ssh /ipfs /ip4/127.0.0.1/tcp/22 +ipfs p2p forward /x/ssh /ipfs /ip4/127.0.0.1/tcp/22 ``` ***Then, on "client" node:*** ```sh -ipfs p2p forward /p2p/ssh /ip4/127.0.0.1/tcp/2222 /ipfs/$SERVER_ID +ipfs p2p forward /x/ssh /ip4/127.0.0.1/tcp/2222 /ipfs/$SERVER_ID ``` You should now be able to connect to your ssh server through a libp2p connection diff --git a/test/sharness/t0180-p2p.sh b/test/sharness/t0180-p2p.sh index 940ce109211..ce639991886 100755 --- a/test/sharness/t0180-p2p.sh +++ b/test/sharness/t0180-p2p.sh @@ -38,7 +38,7 @@ test_expect_success "enable filestore config setting" ' ' test_expect_success 'start p2p listener' ' - ipfsi 0 p2p forward /p2p/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log + ipfsi 0 p2p forward /x/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log ' # Server to client communications @@ -69,7 +69,7 @@ test_server_to_client() { spawn_sending_server test_expect_success 'S->C Setup client side' ' - ipfsi 1 p2p forward /p2p/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/${PEERID_0} 2>&1 > dialer-stdouterr.log + ipfsi 1 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/${PEERID_0} 2>&1 > dialer-stdouterr.log ' test_server_to_client @@ -87,7 +87,7 @@ spawn_sending_server test_server_to_client test_expect_success 'S->C Close local listener' ' - ipfsi 1 p2p close -p /p2p/p2p-test + ipfsi 1 p2p close -p /x/p2p-test ' check_test_ports @@ -102,7 +102,7 @@ test_expect_success 'C->S Spawn receiving server' ' ' test_expect_success 'C->S Setup client side' ' - ipfsi 1 p2p forward /p2p/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/${PEERID_0} 2>&1 > dialer-stdouterr.log + ipfsi 1 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/${PEERID_0} 2>&1 > dialer-stdouterr.log ' test_expect_success 'C->S Connect and receive data' ' @@ -119,7 +119,7 @@ test_expect_success 'C->S Output looks good' ' ' test_expect_success 'C->S Close local listener' ' - ipfsi 1 p2p close -p /p2p/p2p-test + ipfsi 1 p2p close -p /x/p2p-test ' check_test_ports @@ -127,7 +127,7 @@ check_test_ports # Listing streams test_expect_success "'ipfs p2p ls' succeeds" ' - echo "/p2p/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101" > expected && + echo "/x/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101" > expected && ipfsi 0 p2p ls > actual ' @@ -136,7 +136,7 @@ test_expect_success "'ipfs p2p ls' output looks good" ' ' test_expect_success "Cannot re-register app handler" ' - test_must_fail ipfsi 0 p2p forward /p2p/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 + test_must_fail ipfsi 0 p2p forward /x/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 ' test_expect_success "'ipfs p2p stream ls' output is empty" ' @@ -147,7 +147,7 @@ test_expect_success "'ipfs p2p stream ls' output is empty" ' test_expect_success "Setup: Idle stream" ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 & - ipfsi 1 p2p forward /p2p/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && + ipfsi 1 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && ma-pipe-unidir --pidFile=client.pid recv /ip4/127.0.0.1/tcp/10102 & test_wait_for_file 30 100ms listener.pid && @@ -156,7 +156,7 @@ test_expect_success "Setup: Idle stream" ' ' test_expect_success "'ipfs p2p stream ls' succeeds" ' - echo "3 /p2p/p2p-test /ipfs/$PEERID_1 /ip4/127.0.0.1/tcp/10101" > expected + echo "3 /x/p2p-test /ipfs/$PEERID_1 /ip4/127.0.0.1/tcp/10101" > expected ipfsi 0 p2p stream ls > actual ' @@ -172,13 +172,13 @@ test_expect_success "'ipfs p2p stream close' closes stream" ' ' test_expect_success "'ipfs p2p close' closes remote handler" ' - ipfsi 0 p2p close -p /p2p/p2p-test && + ipfsi 0 p2p close -p /x/p2p-test && ipfsi 0 p2p ls > actual && test_must_be_empty actual ' test_expect_success "'ipfs p2p close' closes local handler" ' - ipfsi 1 p2p close -p /p2p/p2p-test && + ipfsi 1 p2p close -p /x/p2p-test && ipfsi 1 p2p ls > actual && test_must_be_empty actual ' @@ -188,8 +188,8 @@ check_test_ports test_expect_success "Setup: Idle stream(2)" ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 & - ipfsi 0 p2p forward /p2p/p2p-test2 /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && - ipfsi 1 p2p forward /p2p/p2p-test2 /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && + ipfsi 0 p2p forward /x/p2p-test2 /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && + ipfsi 1 p2p forward /x/p2p-test2 /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && ma-pipe-unidir --pidFile=client.pid recv /ip4/127.0.0.1/tcp/10102 & test_wait_for_file 30 100ms listener.pid && @@ -198,7 +198,7 @@ test_expect_success "Setup: Idle stream(2)" ' ' test_expect_success "'ipfs p2p stream ls' succeeds(2)" ' - echo "4 /p2p/p2p-test2 /ipfs/$PEERID_1 /ip4/127.0.0.1/tcp/10101" > expected + echo "4 /x/p2p-test2 /ipfs/$PEERID_1 /ip4/127.0.0.1/tcp/10101" > expected ipfsi 0 p2p stream ls > actual test_cmp expected actual ' @@ -225,15 +225,15 @@ test_expect_success "'ipfs p2p stream close -a' closes streams" ' check_test_ports test_expect_success "'ipfs p2p close' closes app numeric handlers" ' - ipfsi 0 p2p forward /p2p/1234 /ipfs /ip4/127.0.0.1/tcp/10101 && - ipfsi 0 p2p close -p /p2p/1234 && + ipfsi 0 p2p forward /x/1234 /ipfs /ip4/127.0.0.1/tcp/10101 && + ipfsi 0 p2p close -p /x/1234 && ipfsi 0 p2p ls > actual && test_must_be_empty actual ' -test_expect_success "non /p2p/ scoped protocols are not allowed" ' - test_must_fail ipfsi 0 p2p forward /its/not/a/p2p/path /ipfs /ip4/127.0.0.1/tcp/10101 2> actual && - echo "Error: protocol name must be within '"'"'/p2p/'"'"' namespace" > expected +test_expect_success "non /x/ scoped protocols are not allowed" ' + test_must_fail ipfsi 0 p2p forward /its/not/a/x/path /ipfs /ip4/127.0.0.1/tcp/10101 2> actual && + echo "Error: protocol name must be within '"'"'/x/'"'"' namespace" > expected test_cmp expected actual ' From eb45436512be3524fa3ae61519c2f970bcf62d42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Jun 2018 02:05:10 +0200 Subject: [PATCH 23/44] p2p: allow-custom-protocol option MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 14 ++++++++++++-- test/sharness/t0180-p2p.sh | 9 +++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index daa904e43b9..f5b8eab9915 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -19,6 +19,7 @@ import ( "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit" ) +// P2PProtoPrefix is the default required prefix for protocol names const P2PProtoPrefix = "/x/" // P2PListenerInfoOutput is output type of ls command @@ -91,6 +92,9 @@ Examples: cmdkit.StringArg("listen-address", true, false, "Listening endpoint."), cmdkit.StringArg("target-address", true, false, "Target endpoint."), }, + Options: []cmdkit.Option{ + cmdkit.BoolOption("allow-custom-protocol", "Don't require /x/ prefix"), + }, Run: func(req cmds.Request, res cmds.Response) { n, err := p2pGetNode(req) if err != nil { @@ -102,8 +106,14 @@ Examples: listen := req.Arguments()[1] target := req.Arguments()[2] - if !strings.HasPrefix(proto, P2PProtoPrefix) { - res.SetError(errors.New("protocol name must be within '" + P2PProtoPrefix + "' namespace"), cmdkit.ErrNormal) + allowCustom, _, err := req.Option("allow-custom-protocol").Bool() + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + if !allowCustom && !strings.HasPrefix(proto, P2PProtoPrefix) { + res.SetError(errors.New("protocol name must be within '"+P2PProtoPrefix+"' namespace"), cmdkit.ErrNormal) return } diff --git a/test/sharness/t0180-p2p.sh b/test/sharness/t0180-p2p.sh index ce639991886..10207a3847f 100755 --- a/test/sharness/t0180-p2p.sh +++ b/test/sharness/t0180-p2p.sh @@ -239,6 +239,15 @@ test_expect_success "non /x/ scoped protocols are not allowed" ' check_test_ports +test_expect_success 'start p2p listener on custom proto' ' + ipfsi 0 p2p forward --allow-custom-protocol /p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && + test_must_be_empty listener-stdouterr.log +' + +test_expect_success 'C->S Close local listener' ' + ipfsi 0 p2p close -p /p2p-test +' + test_expect_success 'stop iptb' ' iptb stop ' From f5ab137fc09b402c5da329fff47b562ae19fce7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 18 Jun 2018 18:57:27 +0200 Subject: [PATCH 24/44] p2p: split forward into 2 commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/commands_test.go | 1 + core/commands/p2p.go | 77 ++++++++++++++++++++++++---------- docs/experimental-features.md | 4 +- test/sharness/t0180-p2p.sh | 10 ++--- 4 files changed, 64 insertions(+), 28 deletions(-) diff --git a/core/commands/commands_test.go b/core/commands/commands_test.go index 4944a32d393..a3bef025fa3 100644 --- a/core/commands/commands_test.go +++ b/core/commands/commands_test.go @@ -163,6 +163,7 @@ func TestCommands(t *testing.T) { "/p2p", "/p2p/close", "/p2p/forward", + "/p2p/listen", "/p2p/ls", "/p2p/stream", "/p2p/stream/close", diff --git a/core/commands/p2p.go b/core/commands/p2p.go index f5b8eab9915..364b55ca730 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -62,6 +62,7 @@ are refined`, "stream": p2pStreamCmd, "forward": p2pForwardCmd, + "listen": p2pListenCmd, "close": p2pCloseCmd, "ls": p2pLsCmd, }, @@ -69,19 +70,14 @@ are refined`, var p2pForwardCmd = &cmds.Command{ Helptext: cmdkit.HelpText{ - Tagline: "Forward connections to or from libp2p services", + Tagline: "Forward connections to libp2p service", ShortDescription: ` Forward connections made to to . specifies the libp2p protocol name to use for libp2p connections and/or handlers. It must be prefixed with '` + P2PProtoPrefix + `'. -To create a libp2p service listener, specify '/ipfs' as - -Examples: - ipfs p2p forward ` + P2PProtoPrefix + `myproto /ipfs /ip4/127.0.0.1/tcp/1234 - - Forward connections to 'myproto' libp2p service to 127.0.0.1:1234 - +Example: ipfs p2p forward ` + P2PProtoPrefix + `myproto /ip4/127.0.0.1/tcp/4567 /ipfs/QmPeer - Forward connections to 127.0.0.1:4567 to '` + P2PProtoPrefix + `myproto' service on /ipfs/QmPeer @@ -117,22 +113,61 @@ Examples: return } - if strings.HasPrefix(listen, "/ipfs") { - if listen != "/ipfs" { - res.SetError(errors.New("only '/ipfs' is allowed as libp2p listen address"), cmdkit.ErrNormal) - return - } + if err := forwardLocal(n.Context(), n.P2P, n.Peerstore, proto, listen, target); err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + res.SetOutput(nil) + }, +} - if err := forwardRemote(n.Context(), n.P2P, proto, target); err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } - } else { - if err := forwardLocal(n.Context(), n.P2P, n.Peerstore, proto, listen, target); err != nil { - res.SetError(err, cmdkit.ErrNormal) - return - } +var p2pListenCmd = &cmds.Command{ + Helptext: cmdkit.HelpText{ + Tagline: "Create libp2p service", + ShortDescription: ` +Create libp2p service and forward connections made to . + + specifies the libp2p handler name. It must be prefixed with '` + P2PProtoPrefix + `'. + +Example: + ipfs p2p listen ` + P2PProtoPrefix + `myproto /ip4/127.0.0.1/tcp/1234 + - Forward connections to 'myproto' libp2p service to 127.0.0.1:1234 + +`, + }, + Arguments: []cmdkit.Argument{ + cmdkit.StringArg("protocol", true, false, "Protocol name."), + cmdkit.StringArg("target-address", true, false, "Target endpoint."), + }, + Options: []cmdkit.Option{ + cmdkit.BoolOption("allow-custom-protocol", "Don't require /x/ prefix"), + }, + Run: func(req cmds.Request, res cmds.Response) { + n, err := p2pGetNode(req) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return } + + proto := req.Arguments()[0] + target := req.Arguments()[1] + + allowCustom, _, err := req.Option("allow-custom-protocol").Bool() + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + if !allowCustom && !strings.HasPrefix(proto, P2PProtoPrefix) { + res.SetError(errors.New("protocol name must be within '"+P2PProtoPrefix+"' namespace"), cmdkit.ErrNormal) + return + } + + if err := forwardRemote(n.Context(), n.P2P, proto, target); err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + res.SetOutput(nil) }, } diff --git a/docs/experimental-features.md b/docs/experimental-features.md index d714e624223..29f9f5e3a9a 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -292,7 +292,7 @@ port `$APP_PORT`. Then, configure the p2p listener by running: ```sh -> ipfs p2p forward /x/kickass/1.0 /ipfs /ip4/127.0.0.1/tcp/$APP_PORT +> ipfs p2p listen /x/kickass/1.0 /ip4/127.0.0.1/tcp/$APP_PORT ``` This will configure IPFS to forward all incoming `/x/kickass/1.0` streams to @@ -341,7 +341,7 @@ _you can get `$SERVER_ID` by running `ipfs id -f "\n"`_ ***First, on the "server" node:*** ```sh -ipfs p2p forward /x/ssh /ipfs /ip4/127.0.0.1/tcp/22 +ipfs p2p listen /x/ssh /ip4/127.0.0.1/tcp/22 ``` ***Then, on "client" node:*** diff --git a/test/sharness/t0180-p2p.sh b/test/sharness/t0180-p2p.sh index 10207a3847f..4ae81daa6da 100755 --- a/test/sharness/t0180-p2p.sh +++ b/test/sharness/t0180-p2p.sh @@ -38,7 +38,7 @@ test_expect_success "enable filestore config setting" ' ' test_expect_success 'start p2p listener' ' - ipfsi 0 p2p forward /x/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log + ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log ' # Server to client communications @@ -136,7 +136,7 @@ test_expect_success "'ipfs p2p ls' output looks good" ' ' test_expect_success "Cannot re-register app handler" ' - test_must_fail ipfsi 0 p2p forward /x/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 + test_must_fail ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 ' test_expect_success "'ipfs p2p stream ls' output is empty" ' @@ -188,7 +188,7 @@ check_test_ports test_expect_success "Setup: Idle stream(2)" ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 & - ipfsi 0 p2p forward /x/p2p-test2 /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && + ipfsi 0 p2p listen /x/p2p-test2 /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && ipfsi 1 p2p forward /x/p2p-test2 /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && ma-pipe-unidir --pidFile=client.pid recv /ip4/127.0.0.1/tcp/10102 & @@ -225,7 +225,7 @@ test_expect_success "'ipfs p2p stream close -a' closes streams" ' check_test_ports test_expect_success "'ipfs p2p close' closes app numeric handlers" ' - ipfsi 0 p2p forward /x/1234 /ipfs /ip4/127.0.0.1/tcp/10101 && + ipfsi 0 p2p listen /x/1234 /ip4/127.0.0.1/tcp/10101 && ipfsi 0 p2p close -p /x/1234 && ipfsi 0 p2p ls > actual && test_must_be_empty actual @@ -240,7 +240,7 @@ test_expect_success "non /x/ scoped protocols are not allowed" ' check_test_ports test_expect_success 'start p2p listener on custom proto' ' - ipfsi 0 p2p forward --allow-custom-protocol /p2p-test /ipfs /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && + ipfsi 0 p2p listen --allow-custom-protocol /p2p-test /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log && test_must_be_empty listener-stdouterr.log ' From a3c84e20ef187492581bcc6cadf8517289f365dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 20 Jun 2018 15:17:34 +0200 Subject: [PATCH 25/44] p2p: refactor review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 138 ++++++++++++++++++++-------------- docs/experimental-features.md | 2 +- p2p/listener.go | 55 ++++++++------ p2p/local.go | 117 +++++++++++++++------------- p2p/p2p.go | 7 +- p2p/remote.go | 68 +++++++++++------ p2p/stream.go | 14 ++-- package.json | 6 ++ test/sharness/t0180-p2p.sh | 24 +++++- 9 files changed, 259 insertions(+), 172 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 364b55ca730..3466bb0bbef 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -14,9 +14,11 @@ import ( core "github.com/ipfs/go-ipfs/core" p2p "github.com/ipfs/go-ipfs/p2p" - pstore "gx/ipfs/QmXauCuJzmzapetmC6W4TuDJLL1yFFrVzSHoWv8YdbmnxH/go-libp2p-peerstore" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" + "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" + pstore "gx/ipfs/QmZR2XWVVBCtbgBWnQhWk2xcQfaR3W8faQPriAiaaj7rsr/go-libp2p-peerstore" "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit" + "gx/ipfs/Qme4QgoVPyQqxVc4G1c2L2wc9TDa6o294rtspGMnBNRujm/go-ipfs-addr" ) // P2PProtoPrefix is the default required prefix for protocol names @@ -98,9 +100,23 @@ Example: return } - proto := req.Arguments()[0] - listen := req.Arguments()[1] - target := req.Arguments()[2] + protoOpt := req.Arguments()[0] + listenOpt := req.Arguments()[1] + targetOpt := req.Arguments()[2] + + proto := protocol.ID(protoOpt) + + listen, err := ma.NewMultiaddr(listenOpt) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + target, err := ipfsaddr.ParseString(targetOpt) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } allowCustom, _, err := req.Option("allow-custom-protocol").Bool() if err != nil { @@ -108,7 +124,7 @@ Example: return } - if !allowCustom && !strings.HasPrefix(proto, P2PProtoPrefix) { + if !allowCustom && !strings.HasPrefix(string(proto), P2PProtoPrefix) { res.SetError(errors.New("protocol name must be within '"+P2PProtoPrefix+"' namespace"), cmdkit.ErrNormal) return } @@ -149,8 +165,16 @@ Example: return } - proto := req.Arguments()[0] - target := req.Arguments()[1] + protoOpt := req.Arguments()[0] + targetOpt := req.Arguments()[1] + + proto := protocol.ID(protoOpt) + + target, err := ma.NewMultiaddr(targetOpt) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } allowCustom, _, err := req.Option("allow-custom-protocol").Bool() if err != nil { @@ -158,7 +182,7 @@ Example: return } - if !allowCustom && !strings.HasPrefix(proto, P2PProtoPrefix) { + if !allowCustom && !strings.HasPrefix(string(proto), P2PProtoPrefix) { res.SetError(errors.New("protocol name must be within '"+P2PProtoPrefix+"' namespace"), cmdkit.ErrNormal) return } @@ -173,39 +197,20 @@ Example: } // forwardRemote forwards libp2p service connections to a manet address -func forwardRemote(ctx context.Context, p *p2p.P2P, proto string, target string) error { - if strings.HasPrefix(target, "/ipfs") { - return errors.New("cannot forward libp2p service connections to another libp2p service") - } - - addr, err := ma.NewMultiaddr(target) - if err != nil { - return err - } - +func forwardRemote(ctx context.Context, p *p2p.P2P, proto protocol.ID, target ma.Multiaddr) error { // TODO: return some info - _, err = p.ForwardRemote(ctx, proto, addr) + _, err := p.ForwardRemote(ctx, proto, target) return err } // forwardLocal forwards local connections to a libp2p service -func forwardLocal(ctx context.Context, p *p2p.P2P, ps pstore.Peerstore, proto string, listen string, target string) error { - bindAddr, err := ma.NewMultiaddr(listen) - if err != nil { - return err - } - - addr, peer, err := ParsePeerParam(target) - if err != nil { - return err - } - +func forwardLocal(ctx context.Context, p *p2p.P2P, ps pstore.Peerstore, proto protocol.ID, bindAddr ma.Multiaddr, addr ipfsaddr.IPFSAddr) error { if addr != nil { - ps.AddAddr(peer, addr, pstore.TempAddrTTL) + ps.AddAddr(addr.ID(), addr.Multiaddr(), pstore.TempAddrTTL) } // TODO: return some info - _, err = p.ForwardLocal(ctx, peer, proto, bindAddr) + _, err := p.ForwardLocal(ctx, addr.ID(), proto, bindAddr) return err } @@ -227,9 +232,9 @@ var p2pLsCmd = &cmds.Command{ for _, listener := range n.P2P.Listeners.Listeners { output.Listeners = append(output.Listeners, P2PListenerInfoOutput{ - Protocol: listener.Protocol(), - ListenAddress: listener.ListenAddress(), - TargetAddress: listener.TargetAddress(), + Protocol: string(listener.Protocol()), + ListenAddress: listener.ListenAddress().String(), + TargetAddress: listener.TargetAddress().String(), }) } @@ -272,8 +277,6 @@ var p2pCloseCmd = &cmds.Command{ cmdkit.StringOption("target-address", "t", "Match target address"), }, Run: func(req cmds.Request, res cmds.Response) { - res.SetOutput(nil) - n, err := p2pGetNode(req) if err != nil { res.SetError(err, cmdkit.ErrNormal) @@ -281,12 +284,26 @@ var p2pCloseCmd = &cmds.Command{ } closeAll, _, _ := req.Option("all").Bool() - proto, p, _ := req.Option("protocol").String() - listen, l, _ := req.Option("listen-address").String() - target, t, _ := req.Option("target-address").String() + protoOpt, p, _ := req.Option("protocol").String() + listenOpt, l, _ := req.Option("listen-address").String() + targetOpt, t, _ := req.Option("target-address").String() + + proto := protocol.ID(protoOpt) + + listen, err := ma.NewMultiaddr(listenOpt) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } + + target, err := ma.NewMultiaddr(targetOpt) + if err != nil { + res.SetError(err, cmdkit.ErrNormal) + return + } if !(closeAll || p || l || t) { - res.SetError(errors.New("no connection matching options given"), cmdkit.ErrNormal) + res.SetError(errors.New("no matching options given"), cmdkit.ErrNormal) return } @@ -296,31 +313,36 @@ var p2pCloseCmd = &cmds.Command{ } match := func(listener p2p.Listener) bool { - out := true - - if p { - out = out && (proto == listener.Protocol()) + if closeAll { + return true } - if l { - out = out && (listen == listener.ListenAddress()) + if p && proto != listener.Protocol() { + return false } - if t { - out = out && (target == listener.TargetAddress()) + if l && !listen.Equal(listener.ListenAddress()) { + return false } - - out = out || closeAll - return out + if t && !target.Equal(listener.TargetAddress()) { + return false + } + return true } - var closed int - for _, listener := range n.P2P.Listeners.Listeners { - if !match(listener) { + todo := make([]p2p.Listener, 0) + n.P2P.Listeners.Lock() + for _, l := range n.P2P.Listeners.Listeners { + if !match(l) { continue } - listener.Close() - closed++ + todo = append(todo, l) } - res.SetOutput(closed) + n.P2P.Listeners.Unlock() + + for _, l := range todo { + l.Close() + } + + res.SetOutput(len(todo)) }, Type: int(0), Marshalers: cmds.MarshalerMap{ diff --git a/docs/experimental-features.md b/docs/experimental-features.md index 29f9f5e3a9a..596b8e7f3b0 100644 --- a/docs/experimental-features.md +++ b/docs/experimental-features.md @@ -357,7 +357,7 @@ with `ssh [user]@127.0.0.1 -p 2222`. ### Road to being a real feature - [ ] Needs more people to use and report on how well it works / fits use cases - [ ] More documentation -- [ ] Support other protocols (e.g, unix domain sockets) +- [ ] Support other protocols (e.g, unix domain sockets, websockets, etc.) --- diff --git a/p2p/listener.go b/p2p/listener.go index 5ba37a6645d..aad3aab53f5 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -3,13 +3,18 @@ package p2p import ( "errors" "sync" + + ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" + "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) // Listener listens for connections and proxies them to a target type Listener interface { - Protocol() string - ListenAddress() string - TargetAddress() string + Protocol() protocol.ID + ListenAddress() ma.Multiaddr + TargetAddress() ma.Multiaddr + + start() error // Close closes the listener. Does not affect child streams Close() error @@ -23,43 +28,49 @@ type listenerKey struct { // ListenerRegistry is a collection of local application proto listeners. type ListenerRegistry struct { + sync.Mutex + Listeners map[listenerKey]Listener - lk sync.Mutex } -func (r *ListenerRegistry) lock(l Listener) error { - r.lk.Lock() +// Register registers listenerInfo into this registry and starts it +func (r *ListenerRegistry) Register(l Listener) error { + r.Lock() if _, ok := r.Listeners[getListenerKey(l)]; ok { - r.lk.Unlock() + r.Unlock() return errors.New("listener already registered") } - return nil -} -func (r *ListenerRegistry) unlock() { - r.lk.Unlock() -} + r.Listeners[getListenerKey(l)] = l -// Register registers listenerInfo in this registry -func (r *ListenerRegistry) Register(l Listener) { - defer r.lk.Unlock() + r.Unlock() - r.Listeners[getListenerKey(l)] = l + if err := l.start(); err != nil { + r.Lock() + defer r.Lock() + + delete(r.Listeners, getListenerKey(l)) + return err + } + + return nil } // Deregister removes p2p listener from this registry -func (r *ListenerRegistry) Deregister(k listenerKey) { - r.lk.Lock() - defer r.lk.Unlock() +func (r *ListenerRegistry) Deregister(k listenerKey) bool { + r.Lock() + defer r.Unlock() + _, ok := r.Listeners[k] delete(r.Listeners, k) + return ok } func getListenerKey(l Listener) listenerKey { return listenerKey{ - proto: l.Protocol(), - listen: l.ListenAddress(), - target: l.TargetAddress(), + proto: string(l.Protocol()), + listen: l.ListenAddress().String(), + target: l.TargetAddress().String(), } } diff --git a/p2p/local.go b/p2p/local.go index e07dbed7521..9ab34a26502 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -2,14 +2,15 @@ package p2p import ( "context" + "errors" "time" - manet "gx/ipfs/QmNqRnejxJxjRroz7buhrjfU8i3yNBLa81hFtmf2pXEffN/go-multiaddr-net" - ma "gx/ipfs/QmUxSEGbv2nmYNnfXi7839wwQqTN3kwQeUxe8dTjZWZs7J/go-multiaddr" - peer "gx/ipfs/QmVf8hTAsLLFtn4WPCRNdnaF2Eag2qTBS6uR8AiHPZARXy/go-libp2p-peer" - net "gx/ipfs/QmXdgNhVEgjLxjUoMs5ViQL7pboAt3Y7V7eGHRiE4qrmTE/go-libp2p-net" - protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - pstore "gx/ipfs/QmZhsmorLpD9kmQ4ynbAu4vbKv2goMUnXazwGA4gnWHDjB/go-libp2p-peerstore" + "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" + ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" + "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer" + tec "gx/ipfs/QmWHgLqrghM9zw77nF6gdvT9ExQ2RB9pLxkd8sDHZf1rWb/go-temp-err-catcher" + "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" + "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) // localListener manet streams and proxies them to libp2p services @@ -27,98 +28,106 @@ type localListener struct { } // ForwardLocal creates new P2P stream to a remote listener -func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto string, bindAddr ma.Multiaddr) (Listener, error) { +func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto protocol.ID, bindAddr ma.Multiaddr) (Listener, error) { listener := &localListener{ ctx: ctx, p2p: p2p, id: p2p.identity, - proto: protocol.ID(proto), + proto: proto, laddr: bindAddr, peer: peer, } - if err := p2p.Listeners.lock(listener); err != nil { + if err := p2p.Listeners.Register(listener); err != nil { return nil, err } - maListener, err := manet.Listen(bindAddr) - if err != nil { - p2p.Listeners.unlock() - return nil, err - } - - listener.listener = maListener - - p2p.Listeners.Register(listener) go listener.acceptConns() return listener, nil } -func (l *localListener) dial() (net.Stream, error) { - ctx, cancel := context.WithTimeout(l.ctx, time.Second*30) //TODO: configurable? +func (l *localListener) dial(ctx context.Context) (net.Stream, error) { + cctx, cancel := context.WithTimeout(ctx, time.Second*30) //TODO: configurable? defer cancel() - err := l.p2p.peerHost.Connect(ctx, pstore.PeerInfo{ID: l.peer}) - if err != nil { - return nil, err - } - - return l.p2p.peerHost.NewStream(l.ctx, l.peer, l.proto) + return l.p2p.peerHost.NewStream(cctx, l.peer, l.proto) } func (l *localListener) acceptConns() { for { local, err := l.listener.Accept() if err != nil { + if tec.ErrIsTemporary(err) { + continue + } return } - remote, err := l.dial() - if err != nil { - local.Close() - return - } + go l.setupStream(local) + } +} - tgt, err := ma.NewMultiaddr(l.TargetAddress()) - if err != nil { - local.Close() - return - } +func (l *localListener) setupStream(local manet.Conn) { + remote, err := l.dial(l.ctx) + if err != nil { + local.Close() + log.Warningf("failed to dial to remote %s/%s", l.peer.Pretty(), l.proto) + return + } - stream := &Stream{ - Protocol: l.proto, + stream := &Stream{ + Protocol: l.proto, - OriginAddr: local.RemoteMultiaddr(), - TargetAddr: tgt, + OriginAddr: local.RemoteMultiaddr(), + TargetAddr: l.TargetAddress(), - Local: local, - Remote: remote, + Local: local, + Remote: remote, - Registry: l.p2p.Streams, - } + Registry: l.p2p.Streams, + } + + l.p2p.Streams.Register(stream) + stream.startStreaming() +} - l.p2p.Streams.Register(stream) - stream.startStreaming() +func (l *localListener) start() error { + maListener, err := manet.Listen(l.laddr) + if err != nil { + return err } + + l.listener = maListener + return nil } func (l *localListener) Close() error { - l.listener.Close() - l.p2p.Listeners.Deregister(getListenerKey(l)) + if l.listener == nil { + return errors.New("uninitialized") + } + + if l.p2p.Listeners.Deregister(getListenerKey(l)) { + l.listener.Close() + l.listener = nil + } return nil } -func (l *localListener) Protocol() string { - return string(l.proto) +func (l *localListener) Protocol() protocol.ID { + return l.proto } -func (l *localListener) ListenAddress() string { - return l.laddr.String() +func (l *localListener) ListenAddress() ma.Multiaddr { + return l.laddr } -func (l *localListener) TargetAddress() string { - return "/ipfs/" + l.peer.Pretty() +func (l *localListener) TargetAddress() ma.Multiaddr { + addr, err := ma.NewMultiaddr(maPrefix + l.peer.Pretty()) + if err != nil { + panic(err) + } + return addr } diff --git a/p2p/p2p.go b/p2p/p2p.go index 16a66b58a01..40d82c37543 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -1,13 +1,14 @@ package p2p import ( - "sync" - + logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" pstore "gx/ipfs/QmZR2XWVVBCtbgBWnQhWk2xcQfaR3W8faQPriAiaaj7rsr/go-libp2p-peerstore" p2phost "gx/ipfs/Qmb8T6YBBsjYsVGfrihQLfCJveczZnneSBqBKkYEBWDjge/go-libp2p-host" peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer" ) +var log = logging.Logger("p2p-mount") + // P2P structure holds information on currently running streams/listeners type P2P struct { Listeners *ListenerRegistry @@ -27,11 +28,9 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) Listeners: &ListenerRegistry{ Listeners: map[listenerKey]Listener{}, - lk: sync.Mutex{}, }, Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, - lk: sync.Mutex{}, }, } } diff --git a/p2p/remote.go b/p2p/remote.go index dc78612d9fa..cd6bc1f0c74 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -2,13 +2,16 @@ package p2p import ( "context" + "errors" - manet "gx/ipfs/QmNqRnejxJxjRroz7buhrjfU8i3yNBLa81hFtmf2pXEffN/go-multiaddr-net" - ma "gx/ipfs/QmUxSEGbv2nmYNnfXi7839wwQqTN3kwQeUxe8dTjZWZs7J/go-multiaddr" - net "gx/ipfs/QmXdgNhVEgjLxjUoMs5ViQL7pboAt3Y7V7eGHRiE4qrmTE/go-libp2p-net" + manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" + ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" + net "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) +var maPrefix = "/" + ma.ProtocolWithCode(ma.P_IPFS).Name + "/" + // remoteListener accepts libp2p streams and proxies them to a manet host type remoteListener struct { p2p *P2P @@ -18,70 +21,85 @@ type remoteListener struct { // Address to proxy the incoming connections to addr ma.Multiaddr + + initialized bool } // ForwardRemote creates new p2p listener -func (p2p *P2P) ForwardRemote(ctx context.Context, proto string, addr ma.Multiaddr) (Listener, error) { +func (p2p *P2P) ForwardRemote(ctx context.Context, proto protocol.ID, addr ma.Multiaddr) (Listener, error) { listener := &remoteListener{ p2p: p2p, - proto: protocol.ID(proto), + proto: proto, addr: addr, } - if err := p2p.Listeners.lock(listener); err != nil { + if err := p2p.Listeners.Register(listener); err != nil { return nil, err } - p2p.peerHost.SetStreamHandler(listener.proto, func(remote net.Stream) { - local, err := manet.Dial(addr) + return listener, nil +} + +func (l *remoteListener) start() error { + // TODO: handle errors when https://github.com/libp2p/go-libp2p-host/issues/16 will be done + l.p2p.peerHost.SetStreamHandler(l.proto, func(remote net.Stream) { + local, err := manet.Dial(l.addr) if err != nil { remote.Reset() return } - //TODO: review: is there a better way to do this? - peerMa, err := ma.NewMultiaddr("/ipfs/" + remote.Conn().RemotePeer().Pretty()) + peerMa, err := ma.NewMultiaddr(maPrefix + remote.Conn().RemotePeer().Pretty()) if err != nil { remote.Reset() return } stream := &Stream{ - Protocol: listener.proto, + Protocol: l.proto, OriginAddr: peerMa, - TargetAddr: addr, + TargetAddr: l.addr, Local: local, Remote: remote, - Registry: p2p.Streams, + Registry: l.p2p.Streams, } - p2p.Streams.Register(stream) + l.p2p.Streams.Register(stream) stream.startStreaming() }) - p2p.Listeners.Register(listener) - - return listener, nil + l.initialized = true + return nil } -func (l *remoteListener) Protocol() string { - return string(l.proto) +func (l *remoteListener) Protocol() protocol.ID { + return l.proto } -func (l *remoteListener) ListenAddress() string { - return "/ipfs" +func (l *remoteListener) ListenAddress() ma.Multiaddr { + addr, err := ma.NewMultiaddr(maPrefix + l.p2p.identity.Pretty()) + if err != nil { + panic(err) + } + return addr } -func (l *remoteListener) TargetAddress() string { - return l.addr.String() +func (l *remoteListener) TargetAddress() ma.Multiaddr { + return l.addr } func (l *remoteListener) Close() error { - l.p2p.peerHost.RemoveStreamHandler(protocol.ID(l.proto)) - l.p2p.Listeners.Deregister(getListenerKey(l)) + if !l.initialized { + return errors.New("uninitialized") + } + + if l.p2p.Listeners.Deregister(getListenerKey(l)) { + l.p2p.peerHost.RemoveStreamHandler(l.proto) + l.initialized = false + } return nil } diff --git a/p2p/stream.go b/p2p/stream.go index c6e6e398a89..626c842f9e3 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -4,9 +4,9 @@ import ( "io" "sync" - manet "gx/ipfs/QmNqRnejxJxjRroz7buhrjfU8i3yNBLa81hFtmf2pXEffN/go-multiaddr-net" - ma "gx/ipfs/QmUxSEGbv2nmYNnfXi7839wwQqTN3kwQeUxe8dTjZWZs7J/go-multiaddr" - net "gx/ipfs/QmXdgNhVEgjLxjUoMs5ViQL7pboAt3Y7V7eGHRiE4qrmTE/go-libp2p-net" + manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" + ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" + net "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) @@ -43,8 +43,12 @@ func (s *Stream) Reset() error { func (s *Stream) startStreaming() { go func() { - io.Copy(s.Local, s.Remote) - s.Reset() + _, err := io.Copy(s.Local, s.Remote) + if err != nil { + s.Reset() + } else { + s.Close() + } }() go func() { diff --git a/package.json b/package.json index c49cd25abd8..7fb69b818f7 100644 --- a/package.json +++ b/package.json @@ -486,6 +486,12 @@ "name": "go-ipns", "version": "0.1.8" }, + { + "author": "whyrusleeping", + "hash": "QmWHgLqrghM9zw77nF6gdvT9ExQ2RB9pLxkd8sDHZf1rWb", + "name": "go-temp-err-catcher", + "version": "0.0.0" + }, { "author": "why", "hash": "QmVDDgboX5nPUE4pBcK2xC1b9XbStA4t2KrUWBRMr9AiFd", diff --git a/test/sharness/t0180-p2p.sh b/test/sharness/t0180-p2p.sh index 4ae81daa6da..f9edfcc8d50 100755 --- a/test/sharness/t0180-p2p.sh +++ b/test/sharness/t0180-p2p.sh @@ -127,7 +127,7 @@ check_test_ports # Listing streams test_expect_success "'ipfs p2p ls' succeeds" ' - echo "/x/p2p-test /ipfs /ip4/127.0.0.1/tcp/10101" > expected && + echo "/x/p2p-test /ipfs/$PEERID_0 /ip4/127.0.0.1/tcp/10101" > expected && ipfsi 0 p2p ls > actual ' @@ -144,10 +144,12 @@ test_expect_success "'ipfs p2p stream ls' output is empty" ' test_must_be_empty actual ' +check_test_ports + test_expect_success "Setup: Idle stream" ' ma-pipe-unidir --listen --pidFile=listener.pid recv /ip4/127.0.0.1/tcp/10101 & - ipfsi 1 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 2>&1 > dialer-stdouterr.log && + ipfsi 1 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/$PEERID_0 && ma-pipe-unidir --pidFile=client.pid recv /ip4/127.0.0.1/tcp/10102 & test_wait_for_file 30 100ms listener.pid && @@ -231,8 +233,22 @@ test_expect_success "'ipfs p2p close' closes app numeric handlers" ' test_must_be_empty actual ' +test_expect_success "'ipfs p2p close' closes by listen addr" ' + ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 && + ipfsi 0 p2p close -l /ipfs/$PEERID_0 && + ipfsi 0 p2p ls > actual && + test_must_be_empty actual +' + +test_expect_success "'ipfs p2p close' closes by target addr" ' + ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 && + ipfsi 0 p2p close -t /ip4/127.0.0.1/tcp/10101 && + ipfsi 0 p2p ls > actual && + test_must_be_empty actual +' + test_expect_success "non /x/ scoped protocols are not allowed" ' - test_must_fail ipfsi 0 p2p forward /its/not/a/x/path /ipfs /ip4/127.0.0.1/tcp/10101 2> actual && + test_must_fail ipfsi 0 p2p listen /its/not/a/x/path /ip4/127.0.0.1/tcp/10101 2> actual && echo "Error: protocol name must be within '"'"'/x/'"'"' namespace" > expected test_cmp expected actual ' @@ -246,6 +262,8 @@ test_expect_success 'start p2p listener on custom proto' ' test_expect_success 'C->S Close local listener' ' ipfsi 0 p2p close -p /p2p-test + ipfsi 0 p2p ls > actual && + test_must_be_empty actual ' test_expect_success 'stop iptb' ' From 4c98edaff6b392962ff863b79d92a56df57a7ec3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 25 Jun 2018 17:25:35 +0200 Subject: [PATCH 26/44] p2p: fix remote/local listener races MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/listener.go | 27 +++++++++++++++++++-------- p2p/local.go | 9 ++++----- p2p/p2p.go | 1 + p2p/remote.go | 13 ++++--------- 4 files changed, 28 insertions(+), 22 deletions(-) diff --git a/p2p/listener.go b/p2p/listener.go index aad3aab53f5..c78a6d61551 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -31,26 +31,33 @@ type ListenerRegistry struct { sync.Mutex Listeners map[listenerKey]Listener + starting map[listenerKey]struct{} } // Register registers listenerInfo into this registry and starts it func (r *ListenerRegistry) Register(l Listener) error { r.Lock() + k := getListenerKey(l) - if _, ok := r.Listeners[getListenerKey(l)]; ok { + if _, ok := r.Listeners[k]; ok { r.Unlock() return errors.New("listener already registered") } - r.Listeners[getListenerKey(l)] = l + r.Listeners[k] = l + r.starting[k] = struct{}{} r.Unlock() - if err := l.start(); err != nil { - r.Lock() - defer r.Lock() + err := l.start() - delete(r.Listeners, getListenerKey(l)) + r.Lock() + defer r.Unlock() + + delete(r.starting, k) + + if err != nil { + delete(r.Listeners, k) return err } @@ -58,13 +65,17 @@ func (r *ListenerRegistry) Register(l Listener) error { } // Deregister removes p2p listener from this registry -func (r *ListenerRegistry) Deregister(k listenerKey) bool { +func (r *ListenerRegistry) Deregister(k listenerKey) (bool, error) { r.Lock() defer r.Unlock() + if _, ok := r.starting[k]; ok { + return false, errors.New("listener didn't start yet") + } + _, ok := r.Listeners[k] delete(r.Listeners, k) - return ok + return ok, nil } func getListenerKey(l Listener) listenerKey { diff --git a/p2p/local.go b/p2p/local.go index 9ab34a26502..4d6c8a89447 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -2,7 +2,6 @@ package p2p import ( "context" - "errors" "time" "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" @@ -105,11 +104,11 @@ func (l *localListener) start() error { } func (l *localListener) Close() error { - if l.listener == nil { - return errors.New("uninitialized") + ok, err := l.p2p.Listeners.Deregister(getListenerKey(l)) + if err != nil { + return err } - - if l.p2p.Listeners.Deregister(getListenerKey(l)) { + if ok { l.listener.Close() l.listener = nil } diff --git a/p2p/p2p.go b/p2p/p2p.go index 40d82c37543..8228e97c88d 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -28,6 +28,7 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) Listeners: &ListenerRegistry{ Listeners: map[listenerKey]Listener{}, + starting: map[listenerKey]struct{}{}, }, Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, diff --git a/p2p/remote.go b/p2p/remote.go index cd6bc1f0c74..23161bedc6f 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -2,7 +2,6 @@ package p2p import ( "context" - "errors" manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" @@ -21,8 +20,6 @@ type remoteListener struct { // Address to proxy the incoming connections to addr ma.Multiaddr - - initialized bool } // ForwardRemote creates new p2p listener @@ -72,7 +69,6 @@ func (l *remoteListener) start() error { stream.startStreaming() }) - l.initialized = true return nil } @@ -93,13 +89,12 @@ func (l *remoteListener) TargetAddress() ma.Multiaddr { } func (l *remoteListener) Close() error { - if !l.initialized { - return errors.New("uninitialized") + ok, err := l.p2p.Listeners.Deregister(getListenerKey(l)) + if err != nil { + return err } - - if l.p2p.Listeners.Deregister(getListenerKey(l)) { + if ok { l.p2p.peerHost.RemoveStreamHandler(l.proto) - l.initialized = false } return nil } From 5e8725dbdeacf5c2899c76c3b36152404acd83a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 25 Jun 2018 18:03:20 +0200 Subject: [PATCH 27/44] p2p: test to ensure closing right linsters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 9 ++++++++- p2p/local.go | 3 +-- test/sharness/t0180-p2p.sh | 19 +++++++++++++++---- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 3466bb0bbef..cfa710023ba 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -338,8 +338,15 @@ var p2pCloseCmd = &cmds.Command{ } n.P2P.Listeners.Unlock() + var errs []string for _, l := range todo { - l.Close() + if err := l.Close(); err != nil { + errs = append(errs, err.Error()) + } + } + if len(errs) != 0 { + res.SetError(fmt.Errorf("errors when closing streams: %s", strings.Join(errs, "; ")), cmdkit.ErrNormal) + return } res.SetOutput(len(todo)) diff --git a/p2p/local.go b/p2p/local.go index 4d6c8a89447..c2daa2df12d 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -109,8 +109,7 @@ func (l *localListener) Close() error { return err } if ok { - l.listener.Close() - l.listener = nil + return l.listener.Close() } return nil } diff --git a/test/sharness/t0180-p2p.sh b/test/sharness/t0180-p2p.sh index f9edfcc8d50..e1428c1f27e 100755 --- a/test/sharness/t0180-p2p.sh +++ b/test/sharness/t0180-p2p.sh @@ -233,16 +233,27 @@ test_expect_success "'ipfs p2p close' closes app numeric handlers" ' test_must_be_empty actual ' -test_expect_success "'ipfs p2p close' closes by listen addr" ' +test_expect_success "'ipfs p2p close' closes by target addr" ' ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 && - ipfsi 0 p2p close -l /ipfs/$PEERID_0 && + ipfsi 0 p2p close -t /ip4/127.0.0.1/tcp/10101 && ipfsi 0 p2p ls > actual && test_must_be_empty actual ' -test_expect_success "'ipfs p2p close' closes by target addr" ' +test_expect_success "'ipfs p2p close' closes right listeners" ' ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 && - ipfsi 0 p2p close -t /ip4/127.0.0.1/tcp/10101 && + ipfsi 0 p2p forward /x/p2p-test /ip4/127.0.0.1/tcp/10101 /ipfs/$PEERID_1 && + echo "/x/p2p-test /ipfs/$PEERID_0 /ip4/127.0.0.1/tcp/10101" > expected && + + ipfsi 0 p2p close -l /ip4/127.0.0.1/tcp/10101 && + ipfsi 0 p2p ls > actual && + test_cmp expected actual +' + +check_test_ports + +test_expect_success "'ipfs p2p close' closes by listen addr" ' + ipfsi 0 p2p close -l /ipfs/$PEERID_0 && ipfsi 0 p2p ls > actual && test_must_be_empty actual ' From 3c6a168d16274051bcab9f2c1c8b30b60cf74587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 13 Jul 2018 10:22:09 +0200 Subject: [PATCH 28/44] p2p: fix imports after rebase MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index cfa710023ba..e316639ce8b 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -14,6 +14,7 @@ import ( core "github.com/ipfs/go-ipfs/core" p2p "github.com/ipfs/go-ipfs/p2p" + "gx/ipfs/Qme4QgoVPyQqxVc4G1c2L2wc9TDa6o294rtspGMnBNRujm/go-ipfs-addr" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" pstore "gx/ipfs/QmZR2XWVVBCtbgBWnQhWk2xcQfaR3W8faQPriAiaaj7rsr/go-libp2p-peerstore" From 8849193de0f0d11c792f01b3d309401f488fd5d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 13 Jul 2018 16:40:11 +0200 Subject: [PATCH 29/44] p2p: more locks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 13 ++++++++++++- p2p/stream.go | 14 +++++++------- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index e316639ce8b..8d01b26070d 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -231,6 +231,7 @@ var p2pLsCmd = &cmds.Command{ output := &P2PLsOutput{} + n.P2P.Listeners.Lock() for _, listener := range n.P2P.Listeners.Listeners { output.Listeners = append(output.Listeners, P2PListenerInfoOutput{ Protocol: string(listener.Protocol()), @@ -238,6 +239,7 @@ var p2pLsCmd = &cmds.Command{ TargetAddress: listener.TargetAddress().String(), }) } + n.P2P.Listeners.Unlock() res.SetOutput(output) }, @@ -402,6 +404,7 @@ var p2pStreamLsCmd = &cmds.Command{ output := &P2PStreamsOutput{} + n.P2P.Streams.Lock() for id, s := range n.P2P.Streams.Streams { output.Streams = append(output.Streams, P2PStreamInfoOutput{ HandlerID: strconv.FormatUint(id, 10), @@ -412,6 +415,7 @@ var p2pStreamLsCmd = &cmds.Command{ TargetAddress: s.TargetAddr.String(), }) } + n.P2P.Streams.Unlock() res.SetOutput(output) }, @@ -476,15 +480,22 @@ var p2pStreamCloseCmd = &cmds.Command{ } } + toClose := make([]*p2p.Stream, 0, 1) + n.P2P.Streams.Lock() for id, stream := range n.P2P.Streams.Streams { if !closeAll && handlerID != id { continue } - stream.Reset() + toClose = append(toClose, stream) if !closeAll { break } } + n.P2P.Streams.Unlock() + + for _, s := range toClose { + s.Reset() + } }, } diff --git a/p2p/stream.go b/p2p/stream.go index 626c842f9e3..26e70ad7683 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -63,16 +63,16 @@ func (s *Stream) startStreaming() { // StreamRegistry is a collection of active incoming and outgoing proto app streams. type StreamRegistry struct { - Streams map[uint64]*Stream - lk sync.Mutex + sync.Mutex - nextID uint64 + Streams map[uint64]*Stream + nextID uint64 } // Register registers a stream to the registry func (r *StreamRegistry) Register(streamInfo *Stream) { - r.lk.Lock() - defer r.lk.Unlock() + r.Lock() + defer r.Unlock() streamInfo.id = r.nextID r.Streams[r.nextID] = streamInfo @@ -81,8 +81,8 @@ func (r *StreamRegistry) Register(streamInfo *Stream) { // Deregister deregisters stream from the registry func (r *StreamRegistry) Deregister(streamID uint64) { - r.lk.Lock() - defer r.lk.Unlock() + r.Lock() + defer r.Unlock() delete(r.Streams, streamID) } From 4badcdc340f809046966fad75707bbb2182151c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 30 Jul 2018 17:15:01 +0200 Subject: [PATCH 30/44] p2p: tag connections in connection manager MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 1 - p2p/local.go | 13 ++++++++++--- p2p/p2p.go | 2 +- p2p/remote.go | 13 +++++++++++-- p2p/stream.go | 7 ++++++- 5 files changed, 28 insertions(+), 8 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 8d01b26070d..301195f4abb 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -14,7 +14,6 @@ import ( core "github.com/ipfs/go-ipfs/core" p2p "github.com/ipfs/go-ipfs/p2p" - "gx/ipfs/Qme4QgoVPyQqxVc4G1c2L2wc9TDa6o294rtspGMnBNRujm/go-ipfs-addr" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" pstore "gx/ipfs/QmZR2XWVVBCtbgBWnQhWk2xcQfaR3W8faQPriAiaaj7rsr/go-libp2p-peerstore" diff --git a/p2p/local.go b/p2p/local.go index c2daa2df12d..e64c7de0b46 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -4,12 +4,12 @@ import ( "context" "time" + "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" - ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" - "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer" tec "gx/ipfs/QmWHgLqrghM9zw77nF6gdvT9ExQ2RB9pLxkd8sDHZf1rWb/go-temp-err-catcher" - "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" + ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" + "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer" ) // localListener manet streams and proxies them to libp2p services @@ -77,6 +77,9 @@ func (l *localListener) setupStream(local manet.Conn) { return } + cmgr := l.p2p.peerHost.ConnManager() + cmgr.TagPeer(l.peer, CMGR_TAG, 20) + stream := &Stream{ Protocol: l.proto, @@ -87,6 +90,10 @@ func (l *localListener) setupStream(local manet.Conn) { Remote: remote, Registry: l.p2p.Streams, + + cleanup: func() { + cmgr.UntagPeer(l.peer, CMGR_TAG) + }, } l.p2p.Streams.Register(stream) diff --git a/p2p/p2p.go b/p2p/p2p.go index 8228e97c88d..737bf143436 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -1,9 +1,9 @@ package p2p import ( - logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" pstore "gx/ipfs/QmZR2XWVVBCtbgBWnQhWk2xcQfaR3W8faQPriAiaaj7rsr/go-libp2p-peerstore" p2phost "gx/ipfs/Qmb8T6YBBsjYsVGfrihQLfCJveczZnneSBqBKkYEBWDjge/go-libp2p-host" + logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer" ) diff --git a/p2p/remote.go b/p2p/remote.go index 23161bedc6f..53c25ddf419 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -3,9 +3,9 @@ package p2p import ( "context" + net "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" - net "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) @@ -47,12 +47,17 @@ func (l *remoteListener) start() error { return } - peerMa, err := ma.NewMultiaddr(maPrefix + remote.Conn().RemotePeer().Pretty()) + peer := remote.Conn().RemotePeer() + + peerMa, err := ma.NewMultiaddr(maPrefix + peer.Pretty()) if err != nil { remote.Reset() return } + cmgr := l.p2p.peerHost.ConnManager() + cmgr.TagPeer(peer, CMGR_TAG, 20) + stream := &Stream{ Protocol: l.proto, @@ -63,6 +68,10 @@ func (l *remoteListener) start() error { Remote: remote, Registry: l.p2p.Streams, + + cleanup: func() { + cmgr.UntagPeer(peer, CMGR_TAG) + }, } l.p2p.Streams.Register(stream) diff --git a/p2p/stream.go b/p2p/stream.go index 26e70ad7683..daa4a3f8504 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -4,12 +4,14 @@ import ( "io" "sync" + net "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" - net "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) +const CMGR_TAG = "stream-fwd" + // Stream holds information on active incoming and outgoing p2p streams. type Stream struct { id uint64 @@ -23,12 +25,15 @@ type Stream struct { Remote net.Stream Registry *StreamRegistry + + cleanup func() } // Close closes stream endpoints and deregisters it func (s *Stream) Close() error { s.Local.Close() s.Remote.Close() + s.cleanup() s.Registry.Deregister(s.id) return nil } From 0f9d28442159e4d4b6645708d1a0fc36f47e7a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 3 Sep 2018 11:41:14 +0200 Subject: [PATCH 31/44] p2p: use host.SetStreamHandlerMatch for now MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/listener.go | 39 +++++++++++++++++++++++++++++++- p2p/p2p.go | 5 +---- p2p/remote.go | 59 ++++++++++++++++++++++++------------------------- 3 files changed, 68 insertions(+), 35 deletions(-) diff --git a/p2p/listener.go b/p2p/listener.go index c78a6d61551..13e70207a22 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -4,8 +4,11 @@ import ( "errors" "sync" + net "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" + p2phost "gx/ipfs/Qmb8T6YBBsjYsVGfrihQLfCJveczZnneSBqBKkYEBWDjge/go-libp2p-host" + peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer" ) // Listener listens for connections and proxies them to a target @@ -28,12 +31,46 @@ type listenerKey struct { // ListenerRegistry is a collection of local application proto listeners. type ListenerRegistry struct { - sync.Mutex + sync.RWMutex Listeners map[listenerKey]Listener starting map[listenerKey]struct{} } +func newListenerRegistry(id peer.ID, host p2phost.Host) *ListenerRegistry { + reg := &ListenerRegistry{ + Listeners: map[listenerKey]Listener{}, + starting: map[listenerKey]struct{}{}, + } + + addr, err := ma.NewMultiaddr(maPrefix + id.Pretty()) + if err != nil { + panic(err) + } + + host.SetStreamHandlerMatch("/x/", func(p string) bool { + reg.RLock() + defer reg.RUnlock() + for _, l := range reg.Listeners { + if l.ListenAddress().Equal(addr) && string(l.Protocol()) == p { + return true + } + } + + return false + }, func(stream net.Stream) { + for _, l := range reg.Listeners { + if l.ListenAddress().Equal(addr) && l.Protocol() == stream.Protocol() { + l.(*remoteListener).handleStream(stream) + } + } + + // panic? + }) + + return reg +} + // Register registers listenerInfo into this registry and starts it func (r *ListenerRegistry) Register(l Listener) error { r.Lock() diff --git a/p2p/p2p.go b/p2p/p2p.go index 737bf143436..a7c0b294eb5 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -26,10 +26,7 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) peerHost: peerHost, peerstore: peerstore, - Listeners: &ListenerRegistry{ - Listeners: map[listenerKey]Listener{}, - starting: map[listenerKey]struct{}{}, - }, + Listeners: newListenerRegistry(identity, peerHost), Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, }, diff --git a/p2p/remote.go b/p2p/remote.go index 53c25ddf419..d713b22eb13 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -39,46 +39,45 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto protocol.ID, addr ma.Mu } func (l *remoteListener) start() error { - // TODO: handle errors when https://github.com/libp2p/go-libp2p-host/issues/16 will be done - l.p2p.peerHost.SetStreamHandler(l.proto, func(remote net.Stream) { - local, err := manet.Dial(l.addr) - if err != nil { - remote.Reset() - return - } + return nil +} - peer := remote.Conn().RemotePeer() +func (l *remoteListener) handleStream(remote net.Stream) { + local, err := manet.Dial(l.addr) + if err != nil { + remote.Reset() + return + } - peerMa, err := ma.NewMultiaddr(maPrefix + peer.Pretty()) - if err != nil { - remote.Reset() - return - } + peer := remote.Conn().RemotePeer() - cmgr := l.p2p.peerHost.ConnManager() - cmgr.TagPeer(peer, CMGR_TAG, 20) + peerMa, err := ma.NewMultiaddr(maPrefix + peer.Pretty()) + if err != nil { + remote.Reset() + return + } - stream := &Stream{ - Protocol: l.proto, + cmgr := l.p2p.peerHost.ConnManager() + cmgr.TagPeer(peer, CMGR_TAG, 20) - OriginAddr: peerMa, - TargetAddr: l.addr, + stream := &Stream{ + Protocol: l.proto, - Local: local, - Remote: remote, + OriginAddr: peerMa, + TargetAddr: l.addr, - Registry: l.p2p.Streams, + Local: local, + Remote: remote, - cleanup: func() { - cmgr.UntagPeer(peer, CMGR_TAG) - }, - } + Registry: l.p2p.Streams, - l.p2p.Streams.Register(stream) - stream.startStreaming() - }) + cleanup: func() { + cmgr.UntagPeer(peer, CMGR_TAG) + }, + } - return nil + l.p2p.Streams.Register(stream) + stream.startStreaming() } func (l *remoteListener) Protocol() protocol.ID { From 35eaa1664c09500e225c7575f9f6448f6ebf6d74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 3 Sep 2018 17:54:55 +0200 Subject: [PATCH 32/44] p2p: rebase updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 6 +++--- p2p/listener.go | 6 +++--- p2p/local.go | 4 ++-- p2p/p2p.go | 8 ++++---- p2p/remote.go | 2 +- p2p/stream.go | 2 +- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 301195f4abb..8769af6cdd8 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -14,11 +14,11 @@ import ( core "github.com/ipfs/go-ipfs/core" p2p "github.com/ipfs/go-ipfs/p2p" + "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - pstore "gx/ipfs/QmZR2XWVVBCtbgBWnQhWk2xcQfaR3W8faQPriAiaaj7rsr/go-libp2p-peerstore" - "gx/ipfs/QmdE4gMduCKCGAcczM2F5ioYDfdeKuPix138wrES1YSr7f/go-ipfs-cmdkit" - "gx/ipfs/Qme4QgoVPyQqxVc4G1c2L2wc9TDa6o294rtspGMnBNRujm/go-ipfs-addr" + pstore "gx/ipfs/QmeKD8YT7887Xu6Z86iZmpYNxrLogJexqxEugSmaf14k64/go-libp2p-peerstore" + "gx/ipfs/Qmen94eLs8WCtaf7ahCg8CncieACAZ4opXJSEsX2U2bDEe/go-ipfs-addr" ) // P2PProtoPrefix is the default required prefix for protocol names diff --git a/p2p/listener.go b/p2p/listener.go index 13e70207a22..b90b377a265 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -4,11 +4,11 @@ import ( "errors" "sync" - net "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" + net "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" + peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - p2phost "gx/ipfs/Qmb8T6YBBsjYsVGfrihQLfCJveczZnneSBqBKkYEBWDjge/go-libp2p-host" - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer" + p2phost "gx/ipfs/QmfH9FKYv3Jp1xiyL8sPchGBUBg6JA6XviwajAo3qgnT3B/go-libp2p-host" ) // Listener listens for connections and proxies them to a target diff --git a/p2p/local.go b/p2p/local.go index e64c7de0b46..c763ebb40ab 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -4,12 +4,12 @@ import ( "context" "time" - "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" + "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" + "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" tec "gx/ipfs/QmWHgLqrghM9zw77nF6gdvT9ExQ2RB9pLxkd8sDHZf1rWb/go-temp-err-catcher" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer" ) // localListener manet streams and proxies them to libp2p services diff --git a/p2p/p2p.go b/p2p/p2p.go index a7c0b294eb5..647a655094a 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -1,10 +1,10 @@ package p2p import ( - pstore "gx/ipfs/QmZR2XWVVBCtbgBWnQhWk2xcQfaR3W8faQPriAiaaj7rsr/go-libp2p-peerstore" - p2phost "gx/ipfs/Qmb8T6YBBsjYsVGfrihQLfCJveczZnneSBqBKkYEBWDjge/go-libp2p-host" - logging "gx/ipfs/QmcVVHfdyv15GVPk7NrxdWjh2hLVccXnoD8j2tyQShiXJb/go-log" - peer "gx/ipfs/QmdVrMn1LhB4ybb8hMVaMLXnA8XRSewMnK6YqXKXoTcRvN/go-libp2p-peer" + peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" + logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log" + pstore "gx/ipfs/QmeKD8YT7887Xu6Z86iZmpYNxrLogJexqxEugSmaf14k64/go-libp2p-peerstore" + p2phost "gx/ipfs/QmfH9FKYv3Jp1xiyL8sPchGBUBg6JA6XviwajAo3qgnT3B/go-libp2p-host" ) var log = logging.Logger("p2p-mount") diff --git a/p2p/remote.go b/p2p/remote.go index d713b22eb13..65f503df77a 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -3,7 +3,7 @@ package p2p import ( "context" - net "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" + net "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" diff --git a/p2p/stream.go b/p2p/stream.go index daa4a3f8504..0443194bebe 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -4,7 +4,7 @@ import ( "io" "sync" - net "gx/ipfs/QmPjvxTpVH8qJyQDnxnsxF9kv9jezKD1kozz1hs3fCGsNh/go-libp2p-net" + net "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" From 87567645f4c39e7d36ebbb604839279eaf948048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 3 Sep 2018 18:29:47 +0200 Subject: [PATCH 33/44] p2p: test for double-registering listeners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/listener.go | 2 -- test/sharness/t0180-p2p.sh | 7 ++++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/p2p/listener.go b/p2p/listener.go index b90b377a265..2205683daf6 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -26,7 +26,6 @@ type Listener interface { type listenerKey struct { proto string listen string - target string } // ListenerRegistry is a collection of local application proto listeners. @@ -119,6 +118,5 @@ func getListenerKey(l Listener) listenerKey { return listenerKey{ proto: string(l.Protocol()), listen: l.ListenAddress().String(), - target: l.TargetAddress().String(), } } diff --git a/test/sharness/t0180-p2p.sh b/test/sharness/t0180-p2p.sh index e1428c1f27e..155373c7cca 100755 --- a/test/sharness/t0180-p2p.sh +++ b/test/sharness/t0180-p2p.sh @@ -23,7 +23,8 @@ test_expect_success 'peer ids' ' check_test_ports() { test_expect_success "test ports are closed" ' (! (netstat -lnp | grep "LISTEN" | grep ":10101 ")) && - (! (netstat -lnp | grep "LISTEN" | grep ":10102 ")) + (! (netstat -lnp | grep "LISTEN" | grep ":10102 "))&& + (! (netstat -lnp | grep "LISTEN" | grep ":10103 ")) ' } check_test_ports @@ -41,6 +42,10 @@ test_expect_success 'start p2p listener' ' ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10101 2>&1 > listener-stdouterr.log ' +test_expect_success 'cannot re-register p2p listener' ' + test_must_fail ipfsi 0 p2p listen /x/p2p-test /ip4/127.0.0.1/tcp/10103 2>&1 > listener-stdouterr.log +' + # Server to client communications spawn_sending_server() { From d184433e6ddf8f18514ede840ce2f825265a704d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 5 Sep 2018 22:13:28 +0200 Subject: [PATCH 34/44] p2p: hold lock when handling new streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/listener.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/p2p/listener.go b/p2p/listener.go index 2205683daf6..c3348618059 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -50,6 +50,7 @@ func newListenerRegistry(id peer.ID, host p2phost.Host) *ListenerRegistry { host.SetStreamHandlerMatch("/x/", func(p string) bool { reg.RLock() defer reg.RUnlock() + for _, l := range reg.Listeners { if l.ListenAddress().Equal(addr) && string(l.Protocol()) == p { return true @@ -58,13 +59,15 @@ func newListenerRegistry(id peer.ID, host p2phost.Host) *ListenerRegistry { return false }, func(stream net.Stream) { + reg.RLock() + defer reg.RUnlock() + for _, l := range reg.Listeners { if l.ListenAddress().Equal(addr) && l.Protocol() == stream.Protocol() { - l.(*remoteListener).handleStream(stream) + go l.(*remoteListener).handleStream(stream) + return } } - - // panic? }) return reg From a5e5b5be66ccc348483f2789afc9adff7c1686a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 5 Sep 2018 22:19:07 +0200 Subject: [PATCH 35/44] p2p: test custom protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- test/sharness/t0180-p2p.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/test/sharness/t0180-p2p.sh b/test/sharness/t0180-p2p.sh index 155373c7cca..9b1f5039378 100755 --- a/test/sharness/t0180-p2p.sh +++ b/test/sharness/t0180-p2p.sh @@ -276,6 +276,14 @@ test_expect_success 'start p2p listener on custom proto' ' test_must_be_empty listener-stdouterr.log ' +spawn_sending_server + +test_expect_success 'S->C Setup client side (custom proto)' ' + ipfsi 1 p2p forward --allow-custom-protocol /p2p-test /ip4/127.0.0.1/tcp/10102 /ipfs/${PEERID_0} 2>&1 > dialer-stdouterr.log +' + +test_server_to_client + test_expect_success 'C->S Close local listener' ' ipfsi 0 p2p close -p /p2p-test ipfsi 0 p2p ls > actual && @@ -286,5 +294,7 @@ test_expect_success 'stop iptb' ' iptb stop ' +check_test_ports + test_done From dd48b8237a402c765e8afb7360336ef194b06893 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 5 Sep 2018 23:06:17 +0200 Subject: [PATCH 36/44] p2p: separate listener types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 34 ++++++-- p2p/local.go | 6 +- p2p/local_listener.go | 113 +++++++++++++++++++++++++++ p2p/p2p.go | 9 ++- p2p/{listener.go => p2p_listener.go} | 34 +++----- p2p/remote.go | 6 +- 6 files changed, 162 insertions(+), 40 deletions(-) create mode 100644 p2p/local_listener.go rename p2p/{listener.go => p2p_listener.go} (75%) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 8769af6cdd8..f8063040c28 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -230,15 +230,25 @@ var p2pLsCmd = &cmds.Command{ output := &P2PLsOutput{} - n.P2P.Listeners.Lock() - for _, listener := range n.P2P.Listeners.Listeners { + n.P2P.ListenersLocal.Lock() + for _, listener := range n.P2P.ListenersLocal.Listeners { output.Listeners = append(output.Listeners, P2PListenerInfoOutput{ Protocol: string(listener.Protocol()), ListenAddress: listener.ListenAddress().String(), TargetAddress: listener.TargetAddress().String(), }) } - n.P2P.Listeners.Unlock() + n.P2P.ListenersLocal.Unlock() + + n.P2P.ListenersP2P.Lock() + for _, listener := range n.P2P.ListenersP2P.Listeners { + output.Listeners = append(output.Listeners, P2PListenerInfoOutput{ + Protocol: string(listener.Protocol()), + ListenAddress: listener.ListenAddress().String(), + TargetAddress: listener.TargetAddress().String(), + }) + } + n.P2P.ListenersP2P.Unlock() res.SetOutput(output) }, @@ -314,7 +324,7 @@ var p2pCloseCmd = &cmds.Command{ return } - match := func(listener p2p.Listener) bool { + match := func(listener p2p.ListenerLocal) bool { if closeAll { return true } @@ -330,15 +340,23 @@ var p2pCloseCmd = &cmds.Command{ return true } - todo := make([]p2p.Listener, 0) - n.P2P.Listeners.Lock() - for _, l := range n.P2P.Listeners.Listeners { + todo := make([]p2p.ListenerLocal, 0) + n.P2P.ListenersLocal.Lock() + for _, l := range n.P2P.ListenersLocal.Listeners { + if !match(l) { + continue + } + todo = append(todo, l) + } + n.P2P.ListenersLocal.Unlock() + n.P2P.ListenersP2P.Lock() + for _, l := range n.P2P.ListenersP2P.Listeners { if !match(l) { continue } todo = append(todo, l) } - n.P2P.Listeners.Unlock() + n.P2P.ListenersP2P.Unlock() var errs []string for _, l := range todo { diff --git a/p2p/local.go b/p2p/local.go index c763ebb40ab..70fd80d1288 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -27,7 +27,7 @@ type localListener struct { } // ForwardLocal creates new P2P stream to a remote listener -func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto protocol.ID, bindAddr ma.Multiaddr) (Listener, error) { +func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto protocol.ID, bindAddr ma.Multiaddr) (ListenerLocal, error) { listener := &localListener{ ctx: ctx, @@ -39,7 +39,7 @@ func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto protocol.I peer: peer, } - if err := p2p.Listeners.Register(listener); err != nil { + if err := p2p.ListenersLocal.Register(listener); err != nil { return nil, err } @@ -111,7 +111,7 @@ func (l *localListener) start() error { } func (l *localListener) Close() error { - ok, err := l.p2p.Listeners.Deregister(getListenerKey(l)) + ok, err := l.p2p.ListenersLocal.Deregister(l.laddr.String()) if err != nil { return err } diff --git a/p2p/local_listener.go b/p2p/local_listener.go new file mode 100644 index 00000000000..88c76ce0646 --- /dev/null +++ b/p2p/local_listener.go @@ -0,0 +1,113 @@ +package p2p + +import ( + "errors" + "sync" + + net "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" + peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" + ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" + "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" + p2phost "gx/ipfs/QmfH9FKYv3Jp1xiyL8sPchGBUBg6JA6XviwajAo3qgnT3B/go-libp2p-host" +) + +// ListenerLocal listens for connections and proxies them to a target +type ListenerLocal interface { + Protocol() protocol.ID + ListenAddress() ma.Multiaddr + TargetAddress() ma.Multiaddr + + start() error + + // Close closes the listener. Does not affect child streams + Close() error +} + +// ListenersLocal is a collection of local application proto listeners. +type ListenersLocal struct { + sync.RWMutex + + Listeners map[string]ListenerLocal + starting map[string]struct{} +} + +func newListenerRegistry(id peer.ID, host p2phost.Host) *ListenersLocal { + reg := &ListenersLocal{ + Listeners: map[string]ListenerLocal{}, + starting: map[string]struct{}{}, + } + + addr, err := ma.NewMultiaddr(maPrefix + id.Pretty()) + if err != nil { + panic(err) + } + + host.SetStreamHandlerMatch("/x/", func(p string) bool { + reg.RLock() + defer reg.RUnlock() + + for _, l := range reg.Listeners { + if l.ListenAddress().Equal(addr) && string(l.Protocol()) == p { + return true + } + } + + return false + }, func(stream net.Stream) { + reg.RLock() + defer reg.RUnlock() + + for _, l := range reg.Listeners { + if l.ListenAddress().Equal(addr) && l.Protocol() == stream.Protocol() { + go l.(*remoteListener).handleStream(stream) + return + } + } + }) + + return reg +} + +// Register registers listenerInfo into this registry and starts it +func (r *ListenersLocal) Register(l ListenerLocal) error { + r.Lock() + k := l.ListenAddress().String() + + if _, ok := r.Listeners[k]; ok { + r.Unlock() + return errors.New("listener already registered") + } + + r.Listeners[k] = l + r.starting[k] = struct{}{} + + r.Unlock() + + err := l.start() + + r.Lock() + defer r.Unlock() + + delete(r.starting, k) + + if err != nil { + delete(r.Listeners, k) + return err + } + + return nil +} + +// Deregister removes p2p listener from this registry +func (r *ListenersLocal) Deregister(k string) (bool, error) { + r.Lock() + defer r.Unlock() + + if _, ok := r.starting[k]; ok { + return false, errors.New("listener didn't start yet") + } + + _, ok := r.Listeners[k] + delete(r.Listeners, k) + return ok, nil +} diff --git a/p2p/p2p.go b/p2p/p2p.go index 647a655094a..7c69d1c26e5 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -11,8 +11,9 @@ var log = logging.Logger("p2p-mount") // P2P structure holds information on currently running streams/listeners type P2P struct { - Listeners *ListenerRegistry - Streams *StreamRegistry + ListenersLocal *ListenersLocal + ListenersP2P *ListenersP2P + Streams *StreamRegistry identity peer.ID peerHost p2phost.Host @@ -26,7 +27,9 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) peerHost: peerHost, peerstore: peerstore, - Listeners: newListenerRegistry(identity, peerHost), + ListenersLocal: newListenerRegistry(identity, peerHost), + ListenersP2P: newListenerP2PRegistry(identity, peerHost), + Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, }, diff --git a/p2p/listener.go b/p2p/p2p_listener.go similarity index 75% rename from p2p/listener.go rename to p2p/p2p_listener.go index c3348618059..7b1d96b8462 100644 --- a/p2p/listener.go +++ b/p2p/p2p_listener.go @@ -12,7 +12,7 @@ import ( ) // Listener listens for connections and proxies them to a target -type Listener interface { +type P2PListener interface { Protocol() protocol.ID ListenAddress() ma.Multiaddr TargetAddress() ma.Multiaddr @@ -23,23 +23,18 @@ type Listener interface { Close() error } -type listenerKey struct { - proto string - listen string -} - // ListenerRegistry is a collection of local application proto listeners. -type ListenerRegistry struct { +type ListenersP2P struct { sync.RWMutex - Listeners map[listenerKey]Listener - starting map[listenerKey]struct{} + Listeners map[protocol.ID]ListenerLocal + starting map[protocol.ID]struct{} } -func newListenerRegistry(id peer.ID, host p2phost.Host) *ListenerRegistry { - reg := &ListenerRegistry{ - Listeners: map[listenerKey]Listener{}, - starting: map[listenerKey]struct{}{}, +func newListenerP2PRegistry(id peer.ID, host p2phost.Host) *ListenersP2P { + reg := &ListenersP2P{ + Listeners: map[protocol.ID]ListenerLocal{}, + starting: map[protocol.ID]struct{}{}, } addr, err := ma.NewMultiaddr(maPrefix + id.Pretty()) @@ -74,10 +69,10 @@ func newListenerRegistry(id peer.ID, host p2phost.Host) *ListenerRegistry { } // Register registers listenerInfo into this registry and starts it -func (r *ListenerRegistry) Register(l Listener) error { +func (r *ListenersP2P) Register(l ListenerLocal) error { r.Lock() - k := getListenerKey(l) + k := l.Protocol() if _, ok := r.Listeners[k]; ok { r.Unlock() return errors.New("listener already registered") @@ -104,7 +99,7 @@ func (r *ListenerRegistry) Register(l Listener) error { } // Deregister removes p2p listener from this registry -func (r *ListenerRegistry) Deregister(k listenerKey) (bool, error) { +func (r *ListenersP2P) Deregister(k protocol.ID) (bool, error) { r.Lock() defer r.Unlock() @@ -116,10 +111,3 @@ func (r *ListenerRegistry) Deregister(k listenerKey) (bool, error) { delete(r.Listeners, k) return ok, nil } - -func getListenerKey(l Listener) listenerKey { - return listenerKey{ - proto: string(l.Protocol()), - listen: l.ListenAddress().String(), - } -} diff --git a/p2p/remote.go b/p2p/remote.go index 65f503df77a..26b401ba01c 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -23,7 +23,7 @@ type remoteListener struct { } // ForwardRemote creates new p2p listener -func (p2p *P2P) ForwardRemote(ctx context.Context, proto protocol.ID, addr ma.Multiaddr) (Listener, error) { +func (p2p *P2P) ForwardRemote(ctx context.Context, proto protocol.ID, addr ma.Multiaddr) (P2PListener, error) { listener := &remoteListener{ p2p: p2p, @@ -31,7 +31,7 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto protocol.ID, addr ma.Mu addr: addr, } - if err := p2p.Listeners.Register(listener); err != nil { + if err := p2p.ListenersP2P.Register(listener); err != nil { return nil, err } @@ -97,7 +97,7 @@ func (l *remoteListener) TargetAddress() ma.Multiaddr { } func (l *remoteListener) Close() error { - ok, err := l.p2p.Listeners.Deregister(getListenerKey(l)) + ok, err := l.p2p.ListenersP2P.Deregister(l.proto) if err != nil { return err } From bba2d05ca951357d9c34bf225419ff15515c0563 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 5 Sep 2018 23:13:01 +0200 Subject: [PATCH 37/44] p2p: cleanup after listener iface split MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/local_listener.go | 32 +------------------------------- p2p/p2p.go | 2 +- p2p/p2p_listener.go | 11 ++++++----- p2p/remote.go | 2 +- 4 files changed, 9 insertions(+), 38 deletions(-) diff --git a/p2p/local_listener.go b/p2p/local_listener.go index 88c76ce0646..64d75f477c3 100644 --- a/p2p/local_listener.go +++ b/p2p/local_listener.go @@ -4,11 +4,9 @@ import ( "errors" "sync" - net "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - p2phost "gx/ipfs/QmfH9FKYv3Jp1xiyL8sPchGBUBg6JA6XviwajAo3qgnT3B/go-libp2p-host" ) // ListenerLocal listens for connections and proxies them to a target @@ -31,40 +29,12 @@ type ListenersLocal struct { starting map[string]struct{} } -func newListenerRegistry(id peer.ID, host p2phost.Host) *ListenersLocal { +func newListenerRegistry(id peer.ID) *ListenersLocal { reg := &ListenersLocal{ Listeners: map[string]ListenerLocal{}, starting: map[string]struct{}{}, } - addr, err := ma.NewMultiaddr(maPrefix + id.Pretty()) - if err != nil { - panic(err) - } - - host.SetStreamHandlerMatch("/x/", func(p string) bool { - reg.RLock() - defer reg.RUnlock() - - for _, l := range reg.Listeners { - if l.ListenAddress().Equal(addr) && string(l.Protocol()) == p { - return true - } - } - - return false - }, func(stream net.Stream) { - reg.RLock() - defer reg.RUnlock() - - for _, l := range reg.Listeners { - if l.ListenAddress().Equal(addr) && l.Protocol() == stream.Protocol() { - go l.(*remoteListener).handleStream(stream) - return - } - } - }) - return reg } diff --git a/p2p/p2p.go b/p2p/p2p.go index 7c69d1c26e5..a20844258ca 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -27,7 +27,7 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) peerHost: peerHost, peerstore: peerstore, - ListenersLocal: newListenerRegistry(identity, peerHost), + ListenersLocal: newListenerRegistry(identity), ListenersP2P: newListenerP2PRegistry(identity, peerHost), Streams: &StreamRegistry{ diff --git a/p2p/p2p_listener.go b/p2p/p2p_listener.go index 7b1d96b8462..7f84c263547 100644 --- a/p2p/p2p_listener.go +++ b/p2p/p2p_listener.go @@ -12,12 +12,13 @@ import ( ) // Listener listens for connections and proxies them to a target -type P2PListener interface { +type ListenerP2P interface { Protocol() protocol.ID ListenAddress() ma.Multiaddr TargetAddress() ma.Multiaddr start() error + handleStream(remote net.Stream) // Close closes the listener. Does not affect child streams Close() error @@ -27,13 +28,13 @@ type P2PListener interface { type ListenersP2P struct { sync.RWMutex - Listeners map[protocol.ID]ListenerLocal + Listeners map[protocol.ID]ListenerP2P starting map[protocol.ID]struct{} } func newListenerP2PRegistry(id peer.ID, host p2phost.Host) *ListenersP2P { reg := &ListenersP2P{ - Listeners: map[protocol.ID]ListenerLocal{}, + Listeners: map[protocol.ID]ListenerP2P{}, starting: map[protocol.ID]struct{}{}, } @@ -59,7 +60,7 @@ func newListenerP2PRegistry(id peer.ID, host p2phost.Host) *ListenersP2P { for _, l := range reg.Listeners { if l.ListenAddress().Equal(addr) && l.Protocol() == stream.Protocol() { - go l.(*remoteListener).handleStream(stream) + go l.handleStream(stream) return } } @@ -69,7 +70,7 @@ func newListenerP2PRegistry(id peer.ID, host p2phost.Host) *ListenersP2P { } // Register registers listenerInfo into this registry and starts it -func (r *ListenersP2P) Register(l ListenerLocal) error { +func (r *ListenersP2P) Register(l ListenerP2P) error { r.Lock() k := l.Protocol() diff --git a/p2p/remote.go b/p2p/remote.go index 26b401ba01c..00971481b61 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -23,7 +23,7 @@ type remoteListener struct { } // ForwardRemote creates new p2p listener -func (p2p *P2P) ForwardRemote(ctx context.Context, proto protocol.ID, addr ma.Multiaddr) (P2PListener, error) { +func (p2p *P2P) ForwardRemote(ctx context.Context, proto protocol.ID, addr ma.Multiaddr) (ListenerP2P, error) { listener := &remoteListener{ p2p: p2p, From c5090508a86c150bb585a0a38f30bfdaadc902c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 6 Sep 2018 00:35:57 +0200 Subject: [PATCH 38/44] p2p: fix connmgr use MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 2 +- p2p/local.go | 8 +----- p2p/p2p.go | 4 ++- p2p/remote.go | 8 +----- p2p/stream.go | 65 ++++++++++++++++++++++++++++++++------------ 5 files changed, 53 insertions(+), 34 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index f8063040c28..ca2b7d73a1d 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -511,7 +511,7 @@ var p2pStreamCloseCmd = &cmds.Command{ n.P2P.Streams.Unlock() for _, s := range toClose { - s.Reset() + n.P2P.Streams.Reset(s) } }, } diff --git a/p2p/local.go b/p2p/local.go index 70fd80d1288..42740d8efc5 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -77,23 +77,17 @@ func (l *localListener) setupStream(local manet.Conn) { return } - cmgr := l.p2p.peerHost.ConnManager() - cmgr.TagPeer(l.peer, CMGR_TAG, 20) - stream := &Stream{ Protocol: l.proto, OriginAddr: local.RemoteMultiaddr(), TargetAddr: l.TargetAddress(), + peer: l.peer, Local: local, Remote: remote, Registry: l.p2p.Streams, - - cleanup: func() { - cmgr.UntagPeer(l.peer, CMGR_TAG) - }, } l.p2p.Streams.Register(stream) diff --git a/p2p/p2p.go b/p2p/p2p.go index a20844258ca..eb773303eae 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -31,7 +31,9 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) ListenersP2P: newListenerP2PRegistry(identity, peerHost), Streams: &StreamRegistry{ - Streams: map[uint64]*Stream{}, + Streams: map[uint64]*Stream{}, + ConnManager: peerHost.ConnManager(), + conns: map[peer.ID]int{}, }, } } diff --git a/p2p/remote.go b/p2p/remote.go index 00971481b61..633497a0cd4 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -57,23 +57,17 @@ func (l *remoteListener) handleStream(remote net.Stream) { return } - cmgr := l.p2p.peerHost.ConnManager() - cmgr.TagPeer(peer, CMGR_TAG, 20) - stream := &Stream{ Protocol: l.proto, OriginAddr: peerMa, TargetAddr: l.addr, + peer: peer, Local: local, Remote: remote, Registry: l.p2p.Streams, - - cleanup: func() { - cmgr.UntagPeer(peer, CMGR_TAG) - }, } l.p2p.Streams.Register(stream) diff --git a/p2p/stream.go b/p2p/stream.go index 0443194bebe..5dc4a95a6d8 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -5,9 +5,11 @@ import ( "sync" net "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" + peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" + ifconnmgr "gx/ipfs/QmVz2p8ZVZ5GcWPNWGs2HZHiZyHumZcJpQdMRpxkMDhc2C/go-libp2p-interface-connmgr" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" - "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" + protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) const CMGR_TAG = "stream-fwd" @@ -20,29 +22,23 @@ type Stream struct { OriginAddr ma.Multiaddr TargetAddr ma.Multiaddr + peer peer.ID Local manet.Conn Remote net.Stream Registry *StreamRegistry - - cleanup func() } -// Close closes stream endpoints and deregisters it -func (s *Stream) Close() error { - s.Local.Close() - s.Remote.Close() - s.cleanup() - s.Registry.Deregister(s.id) +// close closes stream endpoints and deregisters it +func (s *Stream) close() error { + s.Registry.Close(s) return nil } -// Reset closes stream endpoints and deregisters it -func (s *Stream) Reset() error { - s.Local.Close() - s.Remote.Reset() - s.Registry.Deregister(s.id) +// reset closes stream endpoints and deregisters it +func (s *Stream) reset() error { + s.Registry.Reset(s) return nil } @@ -50,18 +46,18 @@ func (s *Stream) startStreaming() { go func() { _, err := io.Copy(s.Local, s.Remote) if err != nil { - s.Reset() + s.reset() } else { - s.Close() + s.close() } }() go func() { _, err := io.Copy(s.Remote, s.Local) if err != nil { - s.Reset() + s.reset() } else { - s.Close() + s.close() } }() } @@ -71,7 +67,10 @@ type StreamRegistry struct { sync.Mutex Streams map[uint64]*Stream + conns map[peer.ID]int nextID uint64 + + ifconnmgr.ConnManager } // Register registers a stream to the registry @@ -79,6 +78,9 @@ func (r *StreamRegistry) Register(streamInfo *Stream) { r.Lock() defer r.Unlock() + r.ConnManager.TagPeer(streamInfo.peer, CMGR_TAG, 20) + r.conns[streamInfo.peer]++ + streamInfo.id = r.nextID r.Streams[r.nextID] = streamInfo r.nextID++ @@ -89,5 +91,32 @@ func (r *StreamRegistry) Deregister(streamID uint64) { r.Lock() defer r.Unlock() + s, ok := r.Streams[streamID] + if !ok { + return + } + p := s.peer + r.conns[p]-- + if r.conns[p] < 1 { + delete(r.conns, p) + r.ConnManager.UntagPeer(p, CMGR_TAG) + } + delete(r.Streams, streamID) } + +// close closes stream endpoints and deregisters it +func (r *StreamRegistry) Close(s *Stream) error { + s.Local.Close() + s.Remote.Close() + s.Registry.Deregister(s.id) + return nil +} + +// reset closes stream endpoints and deregisters it +func (r *StreamRegistry) Reset(s *Stream) error { + s.Local.Close() + s.Remote.Reset() + s.Registry.Deregister(s.id) + return nil +} From 228a71aef271e5a2ce6701d8e8a8a3a60ef3d36b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 6 Sep 2018 11:07:28 +0200 Subject: [PATCH 39/44] p2p: deduplicate some listeners logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 6 +- p2p/{p2p_listener.go => listener.go} | 34 +++++++----- p2p/local.go | 6 +- p2p/local_listener.go | 83 ---------------------------- p2p/p2p.go | 10 ++-- p2p/remote.go | 11 ++-- 6 files changed, 39 insertions(+), 111 deletions(-) rename p2p/{p2p_listener.go => listener.go} (75%) delete mode 100644 p2p/local_listener.go diff --git a/core/commands/p2p.go b/core/commands/p2p.go index ca2b7d73a1d..6695709dcf7 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -324,7 +324,7 @@ var p2pCloseCmd = &cmds.Command{ return } - match := func(listener p2p.ListenerLocal) bool { + match := func(listener p2p.Listener) bool { if closeAll { return true } @@ -340,7 +340,7 @@ var p2pCloseCmd = &cmds.Command{ return true } - todo := make([]p2p.ListenerLocal, 0) + todo := make([]p2p.Listener, 0) n.P2P.ListenersLocal.Lock() for _, l := range n.P2P.ListenersLocal.Listeners { if !match(l) { @@ -389,7 +389,7 @@ var p2pCloseCmd = &cmds.Command{ } /////// -// Listener +// Stream // // p2pStreamCmd is the 'ipfs p2p stream' command diff --git a/p2p/p2p_listener.go b/p2p/listener.go similarity index 75% rename from p2p/p2p_listener.go rename to p2p/listener.go index 7f84c263547..12d9697d3b1 100644 --- a/p2p/p2p_listener.go +++ b/p2p/listener.go @@ -12,30 +12,36 @@ import ( ) // Listener listens for connections and proxies them to a target -type ListenerP2P interface { +type Listener interface { Protocol() protocol.ID ListenAddress() ma.Multiaddr TargetAddress() ma.Multiaddr start() error - handleStream(remote net.Stream) + key() string // Close closes the listener. Does not affect child streams Close() error } -// ListenerRegistry is a collection of local application proto listeners. -type ListenersP2P struct { +type Listeners struct { sync.RWMutex - Listeners map[protocol.ID]ListenerP2P - starting map[protocol.ID]struct{} + Listeners map[string]Listener + starting map[string]struct{} } -func newListenerP2PRegistry(id peer.ID, host p2phost.Host) *ListenersP2P { - reg := &ListenersP2P{ - Listeners: map[protocol.ID]ListenerP2P{}, - starting: map[protocol.ID]struct{}{}, +func newListenersLocal(id peer.ID) *Listeners { + return &Listeners{ + Listeners: map[string]Listener{}, + starting: map[string]struct{}{}, + } +} + +func newListenersP2P(id peer.ID, host p2phost.Host) *Listeners { + reg := &Listeners{ + Listeners: map[string]Listener{}, + starting: map[string]struct{}{}, } addr, err := ma.NewMultiaddr(maPrefix + id.Pretty()) @@ -60,7 +66,7 @@ func newListenerP2PRegistry(id peer.ID, host p2phost.Host) *ListenersP2P { for _, l := range reg.Listeners { if l.ListenAddress().Equal(addr) && l.Protocol() == stream.Protocol() { - go l.handleStream(stream) + go l.(*remoteListener).handleStream(stream) return } } @@ -70,10 +76,10 @@ func newListenerP2PRegistry(id peer.ID, host p2phost.Host) *ListenersP2P { } // Register registers listenerInfo into this registry and starts it -func (r *ListenersP2P) Register(l ListenerP2P) error { +func (r *Listeners) Register(l Listener) error { r.Lock() + k := l.key() - k := l.Protocol() if _, ok := r.Listeners[k]; ok { r.Unlock() return errors.New("listener already registered") @@ -100,7 +106,7 @@ func (r *ListenersP2P) Register(l ListenerP2P) error { } // Deregister removes p2p listener from this registry -func (r *ListenersP2P) Deregister(k protocol.ID) (bool, error) { +func (r *Listeners) Deregister(k string) (bool, error) { r.Lock() defer r.Unlock() diff --git a/p2p/local.go b/p2p/local.go index 42740d8efc5..0ec579a434e 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -27,7 +27,7 @@ type localListener struct { } // ForwardLocal creates new P2P stream to a remote listener -func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto protocol.ID, bindAddr ma.Multiaddr) (ListenerLocal, error) { +func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto protocol.ID, bindAddr ma.Multiaddr) (Listener, error) { listener := &localListener{ ctx: ctx, @@ -130,3 +130,7 @@ func (l *localListener) TargetAddress() ma.Multiaddr { } return addr } + +func (l *localListener) key() string { + return l.ListenAddress().String() +} diff --git a/p2p/local_listener.go b/p2p/local_listener.go deleted file mode 100644 index 64d75f477c3..00000000000 --- a/p2p/local_listener.go +++ /dev/null @@ -1,83 +0,0 @@ -package p2p - -import ( - "errors" - "sync" - - peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" - ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" - "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" -) - -// ListenerLocal listens for connections and proxies them to a target -type ListenerLocal interface { - Protocol() protocol.ID - ListenAddress() ma.Multiaddr - TargetAddress() ma.Multiaddr - - start() error - - // Close closes the listener. Does not affect child streams - Close() error -} - -// ListenersLocal is a collection of local application proto listeners. -type ListenersLocal struct { - sync.RWMutex - - Listeners map[string]ListenerLocal - starting map[string]struct{} -} - -func newListenerRegistry(id peer.ID) *ListenersLocal { - reg := &ListenersLocal{ - Listeners: map[string]ListenerLocal{}, - starting: map[string]struct{}{}, - } - - return reg -} - -// Register registers listenerInfo into this registry and starts it -func (r *ListenersLocal) Register(l ListenerLocal) error { - r.Lock() - k := l.ListenAddress().String() - - if _, ok := r.Listeners[k]; ok { - r.Unlock() - return errors.New("listener already registered") - } - - r.Listeners[k] = l - r.starting[k] = struct{}{} - - r.Unlock() - - err := l.start() - - r.Lock() - defer r.Unlock() - - delete(r.starting, k) - - if err != nil { - delete(r.Listeners, k) - return err - } - - return nil -} - -// Deregister removes p2p listener from this registry -func (r *ListenersLocal) Deregister(k string) (bool, error) { - r.Lock() - defer r.Unlock() - - if _, ok := r.starting[k]; ok { - return false, errors.New("listener didn't start yet") - } - - _, ok := r.Listeners[k] - delete(r.Listeners, k) - return ok, nil -} diff --git a/p2p/p2p.go b/p2p/p2p.go index eb773303eae..031fb076bd2 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -9,10 +9,10 @@ import ( var log = logging.Logger("p2p-mount") -// P2P structure holds information on currently running streams/listeners +// P2P structure holds information on currently running streams/Listeners type P2P struct { - ListenersLocal *ListenersLocal - ListenersP2P *ListenersP2P + ListenersLocal *Listeners + ListenersP2P *Listeners Streams *StreamRegistry identity peer.ID @@ -27,8 +27,8 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) peerHost: peerHost, peerstore: peerstore, - ListenersLocal: newListenerRegistry(identity), - ListenersP2P: newListenerP2PRegistry(identity, peerHost), + ListenersLocal: newListenersLocal(identity), + ListenersP2P: newListenersP2P(identity, peerHost), Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, diff --git a/p2p/remote.go b/p2p/remote.go index 633497a0cd4..1b82727b1c5 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -23,7 +23,7 @@ type remoteListener struct { } // ForwardRemote creates new p2p listener -func (p2p *P2P) ForwardRemote(ctx context.Context, proto protocol.ID, addr ma.Multiaddr) (ListenerP2P, error) { +func (p2p *P2P) ForwardRemote(ctx context.Context, proto protocol.ID, addr ma.Multiaddr) (Listener, error) { listener := &remoteListener{ p2p: p2p, @@ -91,12 +91,13 @@ func (l *remoteListener) TargetAddress() ma.Multiaddr { } func (l *remoteListener) Close() error { - ok, err := l.p2p.ListenersP2P.Deregister(l.proto) + _, err := l.p2p.ListenersP2P.Deregister(string(l.proto)) if err != nil { return err } - if ok { - l.p2p.peerHost.RemoveStreamHandler(l.proto) - } return nil } + +func (l *remoteListener) key() string { + return string(l.proto) +} From 05bc3bd92495b8406d0ee46bf91287f553f4715f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 6 Sep 2018 11:09:56 +0200 Subject: [PATCH 40/44] p2p: simplify remote handler matching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/listener.go | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/p2p/listener.go b/p2p/listener.go index 12d9697d3b1..9aed22ec538 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -44,31 +44,19 @@ func newListenersP2P(id peer.ID, host p2phost.Host) *Listeners { starting: map[string]struct{}{}, } - addr, err := ma.NewMultiaddr(maPrefix + id.Pretty()) - if err != nil { - panic(err) - } - host.SetStreamHandlerMatch("/x/", func(p string) bool { reg.RLock() defer reg.RUnlock() - for _, l := range reg.Listeners { - if l.ListenAddress().Equal(addr) && string(l.Protocol()) == p { - return true - } - } - - return false + _, ok := reg.Listeners[p] + return ok }, func(stream net.Stream) { reg.RLock() defer reg.RUnlock() - for _, l := range reg.Listeners { - if l.ListenAddress().Equal(addr) && l.Protocol() == stream.Protocol() { - go l.(*remoteListener).handleStream(stream) - return - } + l := reg.Listeners[string(stream.Protocol())] + if l != nil { + go l.(*remoteListener).handleStream(stream) } }) From 80b89405edf1054f17244520d619ef7e82cf26c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 6 Sep 2018 11:15:35 +0200 Subject: [PATCH 41/44] p2p: fix docstrings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/listener.go | 2 ++ p2p/stream.go | 12 ++++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/p2p/listener.go b/p2p/listener.go index 9aed22ec538..59df5dde506 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -24,6 +24,8 @@ type Listener interface { Close() error } +// Listeners manages a group of Listener implementations, +// checking for conflicts and optionally dispatching connections type Listeners struct { sync.RWMutex diff --git a/p2p/stream.go b/p2p/stream.go index 5dc4a95a6d8..5a496ec396b 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -12,7 +12,7 @@ import ( protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) -const CMGR_TAG = "stream-fwd" +const cmgrTag = "stream-fwd" // Stream holds information on active incoming and outgoing p2p streams. type Stream struct { @@ -30,7 +30,7 @@ type Stream struct { Registry *StreamRegistry } -// close closes stream endpoints and deregisters it +// close stream endpoints and deregister it func (s *Stream) close() error { s.Registry.Close(s) return nil @@ -78,7 +78,7 @@ func (r *StreamRegistry) Register(streamInfo *Stream) { r.Lock() defer r.Unlock() - r.ConnManager.TagPeer(streamInfo.peer, CMGR_TAG, 20) + r.ConnManager.TagPeer(streamInfo.peer, cmgrTag, 20) r.conns[streamInfo.peer]++ streamInfo.id = r.nextID @@ -99,13 +99,13 @@ func (r *StreamRegistry) Deregister(streamID uint64) { r.conns[p]-- if r.conns[p] < 1 { delete(r.conns, p) - r.ConnManager.UntagPeer(p, CMGR_TAG) + r.ConnManager.UntagPeer(p, cmgrTag) } delete(r.Streams, streamID) } -// close closes stream endpoints and deregisters it +// Close stream endpoints and deregister it func (r *StreamRegistry) Close(s *Stream) error { s.Local.Close() s.Remote.Close() @@ -113,7 +113,7 @@ func (r *StreamRegistry) Close(s *Stream) error { return nil } -// reset closes stream endpoints and deregisters it +// Reset closes stream endpoints and deregisters it func (r *StreamRegistry) Reset(s *Stream) error { s.Local.Close() s.Remote.Reset() From f5cb640d252f6e432e1843e870cfced45a6e648f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 6 Sep 2018 11:42:04 +0200 Subject: [PATCH 42/44] p2p: simplify listener startup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- p2p/listener.go | 36 +++++------------------------------- p2p/local.go | 20 +++++++------------- p2p/p2p.go | 4 ++-- p2p/remote.go | 5 ----- p2p/stream.go | 2 ++ 5 files changed, 16 insertions(+), 51 deletions(-) diff --git a/p2p/listener.go b/p2p/listener.go index 59df5dde506..84db8f62a00 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -5,7 +5,6 @@ import ( "sync" net "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" - peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" p2phost "gx/ipfs/QmfH9FKYv3Jp1xiyL8sPchGBUBg6JA6XviwajAo3qgnT3B/go-libp2p-host" @@ -17,7 +16,6 @@ type Listener interface { ListenAddress() ma.Multiaddr TargetAddress() ma.Multiaddr - start() error key() string // Close closes the listener. Does not affect child streams @@ -30,20 +28,17 @@ type Listeners struct { sync.RWMutex Listeners map[string]Listener - starting map[string]struct{} } -func newListenersLocal(id peer.ID) *Listeners { +func newListenersLocal() *Listeners { return &Listeners{ Listeners: map[string]Listener{}, - starting: map[string]struct{}{}, } } -func newListenersP2P(id peer.ID, host p2phost.Host) *Listeners { +func newListenersP2P(host p2phost.Host) *Listeners { reg := &Listeners{ Listeners: map[string]Listener{}, - starting: map[string]struct{}{}, } host.SetStreamHandlerMatch("/x/", func(p string) bool { @@ -67,31 +62,14 @@ func newListenersP2P(id peer.ID, host p2phost.Host) *Listeners { // Register registers listenerInfo into this registry and starts it func (r *Listeners) Register(l Listener) error { - r.Lock() - k := l.key() - - if _, ok := r.Listeners[k]; ok { - r.Unlock() - return errors.New("listener already registered") - } - - r.Listeners[k] = l - r.starting[k] = struct{}{} - - r.Unlock() - - err := l.start() - r.Lock() defer r.Unlock() - delete(r.starting, k) - - if err != nil { - delete(r.Listeners, k) - return err + if _, ok := r.Listeners[l.key()]; ok { + return errors.New("listener already registered") } + r.Listeners[l.key()] = l return nil } @@ -100,10 +78,6 @@ func (r *Listeners) Deregister(k string) (bool, error) { r.Lock() defer r.Unlock() - if _, ok := r.starting[k]; ok { - return false, errors.New("listener didn't start yet") - } - _, ok := r.Listeners[k] delete(r.Listeners, k) return ok, nil diff --git a/p2p/local.go b/p2p/local.go index 0ec579a434e..010d8099d8c 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -17,7 +17,6 @@ type localListener struct { ctx context.Context p2p *P2P - id peer.ID proto protocol.ID laddr ma.Multiaddr @@ -32,13 +31,19 @@ func (p2p *P2P) ForwardLocal(ctx context.Context, peer peer.ID, proto protocol.I ctx: ctx, p2p: p2p, - id: p2p.identity, proto: proto, laddr: bindAddr, peer: peer, } + maListener, err := manet.Listen(listener.laddr) + if err != nil { + return nil, err + } + + listener.listener = maListener + if err := p2p.ListenersLocal.Register(listener); err != nil { return nil, err } @@ -91,17 +96,6 @@ func (l *localListener) setupStream(local manet.Conn) { } l.p2p.Streams.Register(stream) - stream.startStreaming() -} - -func (l *localListener) start() error { - maListener, err := manet.Listen(l.laddr) - if err != nil { - return err - } - - l.listener = maListener - return nil } func (l *localListener) Close() error { diff --git a/p2p/p2p.go b/p2p/p2p.go index 031fb076bd2..b9924ce03a1 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -27,8 +27,8 @@ func NewP2P(identity peer.ID, peerHost p2phost.Host, peerstore pstore.Peerstore) peerHost: peerHost, peerstore: peerstore, - ListenersLocal: newListenersLocal(identity), - ListenersP2P: newListenersP2P(identity, peerHost), + ListenersLocal: newListenersLocal(), + ListenersP2P: newListenersP2P(peerHost), Streams: &StreamRegistry{ Streams: map[uint64]*Stream{}, diff --git a/p2p/remote.go b/p2p/remote.go index 1b82727b1c5..13ed044a5b5 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -38,10 +38,6 @@ func (p2p *P2P) ForwardRemote(ctx context.Context, proto protocol.ID, addr ma.Mu return listener, nil } -func (l *remoteListener) start() error { - return nil -} - func (l *remoteListener) handleStream(remote net.Stream) { local, err := manet.Dial(l.addr) if err != nil { @@ -71,7 +67,6 @@ func (l *remoteListener) handleStream(remote net.Stream) { } l.p2p.Streams.Register(stream) - stream.startStreaming() } func (l *remoteListener) Protocol() protocol.ID { diff --git a/p2p/stream.go b/p2p/stream.go index 5a496ec396b..ba4632ccf56 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -84,6 +84,8 @@ func (r *StreamRegistry) Register(streamInfo *Stream) { streamInfo.id = r.nextID r.Streams[r.nextID] = streamInfo r.nextID++ + + streamInfo.startStreaming() } // Deregister deregisters stream from the registry From 0b575bc0c541c7d2e30553225d5685cbb8ce1c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Tue, 11 Sep 2018 04:58:39 +0200 Subject: [PATCH 43/44] p2p: rebase import updates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 4 ++-- p2p/listener.go | 4 ++-- p2p/local.go | 2 +- p2p/p2p.go | 4 ++-- p2p/remote.go | 2 +- p2p/stream.go | 4 ++-- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index 6695709dcf7..fe39d8e9098 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -17,8 +17,8 @@ import ( "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - pstore "gx/ipfs/QmeKD8YT7887Xu6Z86iZmpYNxrLogJexqxEugSmaf14k64/go-libp2p-peerstore" - "gx/ipfs/Qmen94eLs8WCtaf7ahCg8CncieACAZ4opXJSEsX2U2bDEe/go-ipfs-addr" + pstore "gx/ipfs/Qmda4cPRvSRyox3SqgJN6DfSZGU5TtHufPTp9uXjFj71X6/go-libp2p-peerstore" + "gx/ipfs/QmePSRaGafvmURQwQkHPDBJsaGwKXC1WpBBHVCQxdr8FPn/go-ipfs-addr" ) // P2PProtoPrefix is the default required prefix for protocol names diff --git a/p2p/listener.go b/p2p/listener.go index 84db8f62a00..b0fa6e714a9 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -4,10 +4,10 @@ import ( "errors" "sync" - net "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" + net "gx/ipfs/QmZNJyx9GGCX4GeuHnLB8fxaxMLs4MjTjHokxfQcCd6Nve/go-libp2p-net" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" - p2phost "gx/ipfs/QmfH9FKYv3Jp1xiyL8sPchGBUBg6JA6XviwajAo3qgnT3B/go-libp2p-host" + p2phost "gx/ipfs/QmeMYW7Nj8jnnEfs9qhm7SxKkoDPUWXu3MsxX6BFwz34tf/go-libp2p-host" ) // Listener listens for connections and proxies them to a target diff --git a/p2p/local.go b/p2p/local.go index 010d8099d8c..ab42fc0c3c7 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -4,11 +4,11 @@ import ( "context" "time" - "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" tec "gx/ipfs/QmWHgLqrghM9zw77nF6gdvT9ExQ2RB9pLxkd8sDHZf1rWb/go-temp-err-catcher" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" + "gx/ipfs/QmZNJyx9GGCX4GeuHnLB8fxaxMLs4MjTjHokxfQcCd6Nve/go-libp2p-net" "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) diff --git a/p2p/p2p.go b/p2p/p2p.go index b9924ce03a1..293b30ee326 100644 --- a/p2p/p2p.go +++ b/p2p/p2p.go @@ -3,8 +3,8 @@ package p2p import ( peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log" - pstore "gx/ipfs/QmeKD8YT7887Xu6Z86iZmpYNxrLogJexqxEugSmaf14k64/go-libp2p-peerstore" - p2phost "gx/ipfs/QmfH9FKYv3Jp1xiyL8sPchGBUBg6JA6XviwajAo3qgnT3B/go-libp2p-host" + pstore "gx/ipfs/Qmda4cPRvSRyox3SqgJN6DfSZGU5TtHufPTp9uXjFj71X6/go-libp2p-peerstore" + p2phost "gx/ipfs/QmeMYW7Nj8jnnEfs9qhm7SxKkoDPUWXu3MsxX6BFwz34tf/go-libp2p-host" ) var log = logging.Logger("p2p-mount") diff --git a/p2p/remote.go b/p2p/remote.go index 13ed044a5b5..63e4848c5b5 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -3,9 +3,9 @@ package p2p import ( "context" - net "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" + net "gx/ipfs/QmZNJyx9GGCX4GeuHnLB8fxaxMLs4MjTjHokxfQcCd6Nve/go-libp2p-net" protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) diff --git a/p2p/stream.go b/p2p/stream.go index ba4632ccf56..0748982fe92 100644 --- a/p2p/stream.go +++ b/p2p/stream.go @@ -4,11 +4,11 @@ import ( "io" "sync" - net "gx/ipfs/QmQSbtGXCyNrj34LWL8EgXyNNYDZ8r3SwQcpW5pPxVhLnM/go-libp2p-net" peer "gx/ipfs/QmQsErDt8Qgw1XrsXf2BpEzDgGWtB1YLsTAARBup5b6B9W/go-libp2p-peer" manet "gx/ipfs/QmV6FjemM1K8oXjrvuq3wuVWWoU2TLDPmNnKrxHzY3v6Ai/go-multiaddr-net" - ifconnmgr "gx/ipfs/QmVz2p8ZVZ5GcWPNWGs2HZHiZyHumZcJpQdMRpxkMDhc2C/go-libp2p-interface-connmgr" + ifconnmgr "gx/ipfs/QmWGGN1nysi1qgqto31bENwESkmZBY4YGK4sZC3qhnqhSv/go-libp2p-interface-connmgr" ma "gx/ipfs/QmYmsdtJ3HsodkePE3eU3TsCaP2YvPZJ4LoXnNkDE5Tpt7/go-multiaddr" + net "gx/ipfs/QmZNJyx9GGCX4GeuHnLB8fxaxMLs4MjTjHokxfQcCd6Nve/go-libp2p-net" protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol" ) From b53936a706905d72711d935180df00df4fd5531c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 12 Sep 2018 23:27:15 +0200 Subject: [PATCH 44/44] p2p: Close on Listeners MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit License: MIT Signed-off-by: Łukasz Magiera --- core/commands/p2p.go | 32 +++----------------------------- p2p/listener.go | 28 ++++++++++++++++++++-------- p2p/local.go | 11 ++--------- p2p/remote.go | 8 +------- test/sharness/t0180-p2p.sh | 6 ++++-- 5 files changed, 30 insertions(+), 55 deletions(-) diff --git a/core/commands/p2p.go b/core/commands/p2p.go index fe39d8e9098..dda18de4c16 100644 --- a/core/commands/p2p.go +++ b/core/commands/p2p.go @@ -340,36 +340,10 @@ var p2pCloseCmd = &cmds.Command{ return true } - todo := make([]p2p.Listener, 0) - n.P2P.ListenersLocal.Lock() - for _, l := range n.P2P.ListenersLocal.Listeners { - if !match(l) { - continue - } - todo = append(todo, l) - } - n.P2P.ListenersLocal.Unlock() - n.P2P.ListenersP2P.Lock() - for _, l := range n.P2P.ListenersP2P.Listeners { - if !match(l) { - continue - } - todo = append(todo, l) - } - n.P2P.ListenersP2P.Unlock() - - var errs []string - for _, l := range todo { - if err := l.Close(); err != nil { - errs = append(errs, err.Error()) - } - } - if len(errs) != 0 { - res.SetError(fmt.Errorf("errors when closing streams: %s", strings.Join(errs, "; ")), cmdkit.ErrNormal) - return - } + done := n.P2P.ListenersLocal.Close(match) + done += n.P2P.ListenersP2P.Close(match) - res.SetOutput(len(todo)) + res.SetOutput(done) }, Type: int(0), Marshalers: cmds.MarshalerMap{ diff --git a/p2p/listener.go b/p2p/listener.go index b0fa6e714a9..e62ab236b3f 100644 --- a/p2p/listener.go +++ b/p2p/listener.go @@ -18,8 +18,8 @@ type Listener interface { key() string - // Close closes the listener. Does not affect child streams - Close() error + // close closes the listener. Does not affect child streams + close() } // Listeners manages a group of Listener implementations, @@ -73,12 +73,24 @@ func (r *Listeners) Register(l Listener) error { return nil } -// Deregister removes p2p listener from this registry -func (r *Listeners) Deregister(k string) (bool, error) { +func (r *Listeners) Close(matchFunc func(listener Listener) bool) int { + todo := make([]Listener, 0) r.Lock() - defer r.Unlock() + for _, l := range r.Listeners { + if !matchFunc(l) { + continue + } + + if _, ok := r.Listeners[l.key()]; ok { + delete(r.Listeners, l.key()) + todo = append(todo, l) + } + } + r.Unlock() + + for _, l := range todo { + l.close() + } - _, ok := r.Listeners[k] - delete(r.Listeners, k) - return ok, nil + return len(todo) } diff --git a/p2p/local.go b/p2p/local.go index ab42fc0c3c7..861ce5e8899 100644 --- a/p2p/local.go +++ b/p2p/local.go @@ -98,15 +98,8 @@ func (l *localListener) setupStream(local manet.Conn) { l.p2p.Streams.Register(stream) } -func (l *localListener) Close() error { - ok, err := l.p2p.ListenersLocal.Deregister(l.laddr.String()) - if err != nil { - return err - } - if ok { - return l.listener.Close() - } - return nil +func (l *localListener) close() { + l.listener.Close() } func (l *localListener) Protocol() protocol.ID { diff --git a/p2p/remote.go b/p2p/remote.go index 63e4848c5b5..924cab4c5ec 100644 --- a/p2p/remote.go +++ b/p2p/remote.go @@ -85,13 +85,7 @@ func (l *remoteListener) TargetAddress() ma.Multiaddr { return l.addr } -func (l *remoteListener) Close() error { - _, err := l.p2p.ListenersP2P.Deregister(string(l.proto)) - if err != nil { - return err - } - return nil -} +func (l *remoteListener) close() {} func (l *remoteListener) key() string { return string(l.proto) diff --git a/test/sharness/t0180-p2p.sh b/test/sharness/t0180-p2p.sh index 9b1f5039378..544cd1f6f6a 100755 --- a/test/sharness/t0180-p2p.sh +++ b/test/sharness/t0180-p2p.sh @@ -285,11 +285,13 @@ test_expect_success 'S->C Setup client side (custom proto)' ' test_server_to_client test_expect_success 'C->S Close local listener' ' - ipfsi 0 p2p close -p /p2p-test - ipfsi 0 p2p ls > actual && + ipfsi 1 p2p close -p /p2p-test + ipfsi 1 p2p ls > actual && test_must_be_empty actual ' +check_test_ports + test_expect_success 'stop iptb' ' iptb stop '