How to print the bytes while the file is being downloaded ? -golang

Question

I'm wondering if it's possible to count and print the number of bytes downloaded while the file is being downloaded.

out, err := os.Create("file.txt")
defer out.Close()
if err != nil {
    fmt.Println(fmt.Sprint(err))
    panic(err)
}
resp, err := http.Get("http://example.com/zip")
defer resp.Body.Close()
if err != nil {
    fmt.Println(fmt.Sprint(err))
    panic(err)
}

n, er := io.Copy(out, resp.Body)
if er != nil {
    fmt.Println(fmt.Sprint(err))
}
fmt.Println(n, "bytes ")

Answer 1

Base @Dave Jack

I add progress value and receiving file data from NC (direct TCP data transfer）

// WriteCounter counts the number of bytes written to it.
type WriteCounter struct {
 Total      int64 // Total # of bytes transferred
 Last       int64
 LastUpdate time.Time
}

// Write implements the io.Writer interface.
//
// Always completes and never returns an error.
func (wc *WriteCounter) Write(p []byte) (int, error) {
  n := len(p)
  wc.Total += int64(n)
  now := time.Now()
  duration := now.Sub(wc.LastUpdate).Seconds()
  if duration > 1 {
    wc.LastUpdate = now

    rate := float64(wc.Total-wc.Last) / (duration) / 1024.0
    wc.Last = wc.Total

    fmt.Printf("Read %d bytes for a total of %d , Rate  %.1fKb/s \n", n, wc.Total, rate)
  }

 return n, nil
}

func Server(dest string) {
  outputFile, err := os.Create(dest)
  if err != nil {
    fmt.Println(err)
  }
  defer outputFile.Close()
  fileWriter := bufio.NewWriter(outputFile)

  serverListener, err := net.Listen("tcp", "0.0.0.0:"+PORT)
  if err != nil {
    fmt.Println(err)
  }
  defer serverListener.Close()

  serverConn, err := serverListener.Accept()
  if err != nil {
    fmt.Println(err)
  }
  defer serverConn.Close()

  wc := &WriteCounter{}
  reader := io.TeeReader(serverConn, wc)
  serverConnReader := bufio.NewReaderSize(reader, 32*1024*1024)

  io.Copy(fileWriter, serverConnReader)
  fileWriter.Flush()
  outputFile.Sync()
  fmt.Println("Done: Writer")
 }

Answer 2

If I understand you correctly, you wish to display the number of bytes read, while the data is transferring. Presumably to maintain some kind of a progress bar or something. In which case, you can use Go's compositional data structures to wrap the reader or writer in a custom io.Reader or io.Writer implementation.

It simply forwards the respective Read or Write call to the underlying stream, while doing some additional work with the (int, error) values returned by them. Here is an example you can run on the Go playground.

package main

import (
    "bytes"
    "fmt"
    "io"
    "os"
    "strings"
)

// PassThru wraps an existing io.Reader.
//
// It simply forwards the Read() call, while displaying
// the results from individual calls to it.
type PassThru struct {
    io.Reader
    total int64 // Total # of bytes transferred
}

// Read 'overrides' the underlying io.Reader's Read method.
// This is the one that will be called by io.Copy(). We simply
// use it to keep track of byte counts and then forward the call.
func (pt *PassThru) Read(p []byte) (int, error) {
    n, err := pt.Reader.Read(p)
    pt.total += int64(n)

    if err == nil {
        fmt.Println("Read", n, "bytes for a total of", pt.total)
    }

    return n, err
}

func main() {
    var src io.Reader    // Source file/url/etc
    var dst bytes.Buffer // Destination file/buffer/etc

    // Create some random input data.
    src = bytes.NewBufferString(strings.Repeat("Some random input data", 1000))

    // Wrap it with our custom io.Reader.
    src = &PassThru{Reader: src}

    count, err := io.Copy(&dst, src)
    if err != nil {
        fmt.Println(err)
        os.Exit(1)
    }

    fmt.Println("Transferred", count, "bytes")
}

The output it generates is this:

Read 512 bytes for a total of 512
Read 1024 bytes for a total of 1536
Read 2048 bytes for a total of 3584
Read 4096 bytes for a total of 7680
Read 8192 bytes for a total of 15872
Read 6128 bytes for a total of 22000
Transferred 22000 bytes

Answer 3

The stdlib now provides something like jimt's PassThru: io.TeeReader. It helps simplify things a bit:

// WriteCounter counts the number of bytes written to it.
type WriteCounter struct {
    Total int64 // Total # of bytes transferred
}

// Write implements the io.Writer interface.  
// 
// Always completes and never returns an error.
func (wc *WriteCounter) Write(p []byte) (int, error) {
    n := len(p)
    wc.Total += int64(n)
    fmt.Printf("Read %d bytes for a total of %d\n", n, wc.Total)
    return n, nil
}

func main() {

    // ...    

    // Wrap it with our custom io.Reader.
    src = io.TeeReader(src, &WriteCounter{})

    // ...
}

playground

Answer 4

Other answers have explained about PassThru. Just provide a full example with callback function base on Dave Jack's answer.

package main

import (
    "fmt"
    "io"
    "net/http"
    "os"
    "strconv"
)

// writeCounter counts the number of bytes written to it.
type writeCounter struct {
    total      int64 // total size
    downloaded int64 // downloaded # of bytes transferred
    onProgress func(downloaded int64, total int64)
}

// Write implements the io.Writer interface.
//
// Always completes and never returns an error.
func (wc *writeCounter) Write(p []byte) (n int, e error) {
    n = len(p)
    wc.downloaded += int64(n)
    wc.onProgress(wc.downloaded, wc.total)
    return
}

func newWriter(size int64, onProgress func(downloaded, total int64)) io.Writer {
    return &writeCounter{total: size, onProgress: onProgress}
}

func main() {
    client := http.DefaultClient
    url := "http://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerFun.mp4"
    saveTo := "/Users/tin/Desktop/ForBiggerFun.mp4"

    download(client, url, saveTo, func(downloaded, total int64) {
        fmt.Printf("Downloaded %d bytes for a total of %d\n", downloaded, total)
    })
}

func download(client *http.Client, url, filePath string, onProgress func(downloaded, total int64)) (err error) {
    // Create file writer
    file, err := os.Create(filePath)
    if err != nil {
        return
    }
    defer file.Close()

    // Determinate the file size
    resp, err := client.Head(url)
    if err != nil {
        return
    }
    contentLength := resp.Header.Get("content-length")
    length, err := strconv.Atoi(contentLength)
    if err != nil {
        return
    }

    // Make request
    resp, err = client.Get(url)
    if err != nil {
        return
    }
    defer resp.Body.Close()

    // pipe stream
    body := io.TeeReader(resp.Body, newWriter(int64(length), onProgress))
    _, err = io.Copy(file, body)
    return err
}

Answer 5

The grab Go package implements progress updates (and many other features) for file downloads.

An example of printing progress updates while a download is in process is included in the following walkthrough: http://cavaliercoder.com/blog/downloading-large-files-in-go.html

You can basically call grab.GetAsync which downloads in a new Go routine and then monitor the BytesTransferred or Progress of the returned grab.Response from the calling thread.