Appending to json file without writing entire file

Question

I have a json which contains one its attributes value as an array and I need to keep appending values to the array and write to a file. Is there a way I could avoid rewrite of the existing data and only append the new values?

----- Moving next question on different thread --------------- what is recommended way for writing big data sets onto the file incremental file write or file dump at the end process?

Answer 1

A general solution makes the most sense if the existing JSON is actually an array, or if it's an object that has an array as the last or only pair, as in your case. Otherwise, you're inserting instead of appending. You probably don't want to read the entire file either.

One approach is not much different than what you were thinking, but handles several details

1. Read the end of the file to verify that it "ends with an array"
2. Retain that part
3. Position the file at that ending array bracket
4. Take the output from a standard encoder for an array of new data, dropping its opening bracket, and inserting a comma if necessary
5. The end of the the new output replaces the original ending array bracket
6. Tack the rest of the tail back on

import (
        "bytes"
        "errors"
        "io"
        "io/ioutil"
        "os"
        "regexp"
        "unicode"
)

const (
        tailCheckLen = 16
)

var (
        arrayEndsObject = regexp.MustCompile("(\\[\\s*)?](\\s*}\\s*)$")
        justArray       = regexp.MustCompile("(\\[\\s*)?](\\s*)$")
)

type jsonAppender struct {
        f               *os.File
        strippedBracket bool
        needsComma      bool
        tail            []byte
}

func (a jsonAppender) Write(b []byte) (int, error) {
        trimmed := 0
        if !a.strippedBracket {
                t := bytes.TrimLeftFunc(b, unicode.IsSpace)
                if len(t) == 0 {
                        return len(b), nil
                }
                if t[0] != '[' {
                        return 0, errors.New("not appending array: " + string(t))
                }
                trimmed = len(b) - len(t) + 1
                b = t[1:]
                a.strippedBracket = true
        }
        if a.needsComma {
                a.needsComma = false
                n, err := a.f.Write([]byte(", "))
                if err != nil {
                        return n, err
                }
        }
        n, err := a.f.Write(b)
        return trimmed + n, err
}

func (a jsonAppender) Close() error {
        if _, err := a.f.Write(a.tail); err != nil {
                defer a.f.Close()
                return err
        }
        return a.f.Close()
}

func JSONArrayAppender(file string) (io.WriteCloser, error) {
        f, err := os.OpenFile(file, os.O_RDWR, 0664)
        if err != nil {
                return nil, err
        }

        pos, err := f.Seek(0, io.SeekEnd)
        if err != nil {
                return nil, err
        }

        if pos < tailCheckLen {
                pos = 0
        } else {
                pos -= tailCheckLen
        }
        _, err = f.Seek(pos, io.SeekStart)
        if err != nil {
                return nil, err
        }

        tail, err := ioutil.ReadAll(f)
        if err != nil {
                return nil, err
        }

        hasElements := false

        if len(tail) == 0 {
                _, err = f.Write([]byte("["))
                if err != nil {
                        return nil, err
                }
        } else {
                var g [][]byte
                if g = arrayEndsObject.FindSubmatch(tail); g != nil {
                } else if g = justArray.FindSubmatch(tail); g != nil {
                } else {
                        return nil, errors.New("does not end with array")
                }

                hasElements = len(g[1]) == 0
                _, err = f.Seek(-int64(len(g[2])+1), io.SeekEnd) // 1 for ]
                if err != nil {
                        return nil, err
                }
                tail = g[2]
        }

        return jsonAppender{f: f, needsComma: hasElements, tail: tail}, nil
}

Usage is then like in this test fragment

    a, err := JSONArrayAppender(f)
    if err != nil {
            t.Fatal(err)
    }

    added := []struct {
            Name string `json:"name"`
    }{
            {"Wonder Woman"},
    }
    if err = json.NewEncoder(a).Encode(added); err != nil {
            t.Fatal(err)
    }

    if err = a.Close(); err != nil {
            t.Fatal(err)
    }

You can use whatever settings on the Encoder you want. The only hard-coded part is handling needsComma, but you can add an argument for that.

Answer 2

If your JSON array is simple you can use something like the following code. In this code, I create JSON array manually.

type item struct {
    Name string
}

func main() {
    fd, err := os.Create("hello.json")
    if err != nil {
        log.Fatal(err)
    }

    fd.Write([]byte{'['})
    for i := 0; i < 10; i++ {
        b, err := json.Marshal(item{
            "parham",
        })
        if err != nil {
            log.Fatal(err)
        }

        if i != 0 {
            fd.Write([]byte{','})
        }
        fd.Write(b)
    }
    fd.Write([]byte{']'})
}

If you want to have a valid array in each step you can write ']' at the end of each iteration and then seek back on the start of the next iteration.