go-libp2p: my nodes don't find any peers with DHT peer discovery

Question

I was basically trying to follow this tutorial https://medium.com/rahasak/libp2p-pubsub-peer-discovery-with-kademlia-dht-c8b131550ac7 but I couldn't get it to work.

First some assumptions, please correct if wrong:

• DHT is required if I want peer discovery over the Internet
• DHT requires bootstrap nodes
• One bootstrap node should be enough for my use case (keep reading)

I am trying to set up a PubSub app together which uses libp2p's gossipSub implementation. I have 2 boxes in the cloud, and on each I would like to run 3 nodes as a test. All ports are open, and nodes can reach other, I have tested this in a non-pubsub (direct) connection setup.

This is my code, basically following the tutorial (I had to change slightly because the latest go libs seem to have changed their API):

I always get FindPeers succeeded: found peers: 0. All nodes seem to be connected to the bootstrap peer, but they don't see each other.

Note: The tutorial uses mDNS for local discovery; I want to use DHT only, as if the app matures, all nodes would be public nodes on the Internet...

Any suggestion?

func NewDHT(ctx context.Context, host host.Host, bootstrapPeers []multiaddr.Multiaddr) (*dht.IpfsDHT, error) {
    var options []dht.Option

    if len(bootstrapPeers) == 0 {
        options = append(options, dht.Mode(dht.ModeServer))
    }
    options = append(options, dht.ProtocolPrefix("/myapp"))

    kdht, err := dht.New(ctx, host, options...)
    if err != nil {
        return nil, err
    }

    if err = kdht.Bootstrap(ctx); err != nil {
        return nil, err
    }

    var wg sync.WaitGroup
    for _, peerAddr := range bootstrapPeers {
        peerinfo, _ := peer.AddrInfoFromP2pAddr(peerAddr)

        wg.Add(1)
        go func() {
            defer wg.Done()
            if err := host.Connect(ctx, *peerinfo); err != nil {
                fmt.Println("connecting failed")
                fmt.Println(err)
            } else {
                fmt.Println("connected to bootstrap")
            }
        }()
    }
    wg.Wait()

    return kdht, nil
}

func Discover(ctx context.Context, h host.Host, dht *dht.IpfsDHT, rendezvous string) {
    var rt = routing.NewRoutingDiscovery(dht)

    ticker1 := time.NewTicker(time.Second * 1)
LOOP:
    for {
        select {
        case <-ctx.Done():
            return
        case <-ticker1.C:
            ttl, err := rt.Advertise(ctx, rendezvous)
            if err != nil {
                fmt.Println("Advertise failed")
                continue
            }
            fmt.Println("Advertise success")
            fmt.Println(ttl)
            break LOOP
        }
    }

    ticker := time.NewTicker(time.Second * 1)
    defer ticker.Stop()

    for {
        select {
        case <-ctx.Done():
            return
        case <-ticker.C:
            peers, err := rt.FindPeers(ctx, rendezvous)
            if err != nil {
                fmt.Println("failed find peers")
                fmt.Println(err)
                continue
            }

            fmt.Println(fmt.Sprintf("FindPeers succeeded: found peers: %d", len(peers)))
            for p := range peers {
                if p.ID == h.ID() {
                    continue
                }
                if h.Network().Connectedness(p.ID) != network.Connected {
                    _, err = h.Network().DialPeer(ctx, p.ID)
                    if err != nil {
                        fmt.Println("failed to connect peer")
                        continue
                    }
                    fmt.Println("connected to peer")
                }
            }
        }
    }
}