Go filepath.Walk hangs

Question

I'm running into a strange problem with filepath.Walk() in go. It starts to run then reaches a point where it just hangs. My CPU sits at 100%, there's no error and it does not continue. I've looked into the directory that I'm traversing and the point where it hangs has nothing special, no symlins or anything like that. Just plain directories and files. Here's the code that I'm using:

type WatchDirs struct {
    Dirs []string
}

//Watches ...Recursively walk the filesystem, entrypoint to file watching
func Watches(tops []string) error {
    var dirs WatchDirs
    for _, top := range tops {
        err := filepath.Walk(top, func(path string, f os.FileInfo, err error) error {
            if err != nil {
                log.Println(err)
                return err
            }
            log.Println("File: ", path)
            if f.IsDir() {
                //log.Println("Path: ", path)
                dirs.Dirs = append(dirs.Dirs, path)
            }
            return nil
        })
        if err != nil {
            log.Println(err)
        }
        log.Println("Continuing Loop")
    }
    log.Println("Starting Dedup: ")
    dirs.Dedup()
    log.Println("Post Dedup: ")
    for _, dir := range dirs.Dirs {
        log.Println(dir)
    }
    dirs.Watch()
    return nil
}

//Dedup ...Remove all duplicate entires from configured directories
func (dirs *WatchDirs) Dedup() {
    log.Println("deduping")
    uniqueSet := make(map[string]bool, len(dirs.Dirs))
    for _, x := range dirs.Dirs {
        uniqueSet[x] = true
    }
    result := make([]string, 0, len(uniqueSet))
    for x := range uniqueSet {
        result = append(result, x)
    }
    dirs.Dirs = result
}

//Watch ...Watch the list of created directories for changes
func (dirs *WatchDirs) Watch() {
    watcher, err := fsnotify.NewWatcher()
    if err != nil {
        log.Fatal(err)
    }
    defer watcher.Close()

    done := make(chan bool)
    go func() {
        for {
            select {
            case event := <-watcher.Events:
                log.Println("event:", event)
                if event.Op&fsnotify.Write == fsnotify.Write {
                    log.Println("modified file:", event.Name)
                }
            case err := <-watcher.Errors:
                log.Println("error:", err)
            }
        }
    }()

    for _, dir := range dirs.Dirs {
        log.Println("Watching dir: ", dir)
        err = watcher.Add(dir)
        if err != nil {
            log.Println(err)
        }
    }
    <-done
}

EDIT: Added additional functions for clarity.

Answer 1

I'm not 100% sure of your intent.. However the grand scheme to me seems that you want to append the path of directories to a slice. You want to check to see if something new has been added and then add that new directory to the slice. Here's my simple version of that. Where every second defined by the ticker I sync paths. I use signals to say, "hey, I want to stop watching for changes and print out all the paths".. I also use a channel to listen to commands from stdin to do other stuff (something you can build on for sure).

I couldn't tell exactly where in your original code the point of failure was. If your watching for changes then the code should be blocking unless you say otherwise.

package main

import (
    "bufio"
    "io"
    "log"
    "os"
    "os/signal"
    "path/filepath"
    "syscall"
    "time"
)

type Watcher struct {
    Dirs    []DirInfo
    Ticker  *time.Ticker
    Stop    chan os.Signal
    Command chan string
}

type DirInfo struct {
    LastMod time.Time
    Path    string
}

func New() *Watcher {
    return &Watcher{
        Stop:    make(chan os.Signal, 1),
        Ticker:  time.NewTicker(1 * time.Second),
        Command: make(chan string),
    }
}

func (w *Watcher) addPath(path string, f os.FileInfo) {
    shouldAppend := true
    for i, d := range w.Dirs {
        if d.Path == path {
            w.Dirs[i].LastMod = f.ModTime()
            shouldAppend = false
            break
        }
    }
    if shouldAppend {
        w.Dirs = append(w.Dirs, DirInfo{f.ModTime(), path})
    }
}

func (w *Watcher) List() {
    if len(w.Dirs) == 0 {
        log.Println("Nothing to show")
        return
    }

    for _, d := range w.Dirs {
        log.Println(d.Path)
    }
    return
}

func (w *Watcher) isNew(path string, f os.FileInfo) bool {
    for _, d := range w.Dirs {
        if path == d.Path {
            t := f.ModTime()
            return t.After(d.LastMod)
        }
    }
    return true
}

func (w *Watcher) Sync(tops []string) {
    for _, top := range tops {
        err := filepath.Walk(top, func(path string, f os.FileInfo, err error) error {
            if err != nil {
                return err
            }
            if f.IsDir() && w.isNew(path, f) {
                w.addPath(path, f)
            }
            return nil
        })

        if err != nil {
            log.Printf("Error %v\n", err)
        }
    }
}


func main() {
    w := New()
    tops := []string{}

    signal.Notify(w.Stop, syscall.SIGINT, syscall.SIGTERM)

    go func() {
        s := bufio.NewScanner(os.Stdin)

        for {
            for s.Scan() {
                w.Command <- s.Text()
            }
        }
    }()

    for {
        select {
        case <-w.Ticker.C:
            w.Sync(tops)
        case <-w.Stop:
            log.Println("signalled to stop")
            w.List()
            os.Exit(0)
        case cmd := <-w.Command:
            switch cmd {
            case "list":
                w.List()
            }
        }
    }

}