Unmarshalling JSON returns an error: invalid character '\x1f' looking for beginning of value

Question

I am trying to write a command line utility that will send a HTTP request, and print the response body in the terminal. The endpoint I'm hitting is supposed to return a JSON body

{"arr":[{"name":"Google","url":"https://www.google.com/"},{"name":"Instagram","url":"https://www.instagram.com/"},{"name":"Pinterest","url":"https://www.pinterest.com/"},{"name":"YouTube","url":"https://www.youtube.com/"}]}

Here is my code, body is the byte array which is the response body

fmt.Println(body)

var dat map[string]interface{}
if err := json.Unmarshal(body, &dat); err != nil {
    panic(err)
}
fmt.Println(dat)

The Println(body) function returns

[31 139 8 0 0 0 0 0 0 3 170 86 74 44 42 82 178 138 174 86 202 75 204 77 85 178 82 114 207 207 79 207 73 85 210 81 42 45 202 81 178 82 202 40 41 
41 40 182 210 215 47 47 47 215 75 7 75 233 37 231 231 234 43 213 234 192 117 120 230 21 151 36 166 23 37 230 98 213 148 9 147 197 208 23 144 153 87 146 90 148 90 92 130 85 95 1 76 22 67 95 100 126 105 72 105 18 118 39 86 230 151 150 148 38 193 220 24 91 11 0 0 0 255 255 3 0 64 164 107 195 223 0 0 0]

and then I get this error

panic: invalid character '\x1f' looking for beginning of value

Am I misunderstanding the json.Unmarshall function? How can I decode that byte array and print the resulting json object to the terminal?

Here is the full code

package main 

import (
    "flag"
    "fmt"
    "os"
    "net"
    "bufio"
    "strings"
    "strconv"
    "net/url"
    "encoding/json"
    "io"
)

func main() {
    
    urlPtr := flag.String("url", "", "URL to fetch. (required)")
    flag.Parse()

    if (*urlPtr == "") {
        flag.PrintDefaults()
        os.Exit(1)
    }

    u, err := url.Parse(*urlPtr)

    if err != nil {
        fmt.Println("Error parsing url")
        os.Exit(1)
    }

    path := u.Path
    hostname := u.Hostname()
    
    conn, err := net.Dial("tcp", hostname + ":80")
    if err != nil {
        fmt.Println("Error setting up tcp connection")
        os.Exit(1)
    }

    if path == "" {
        path = "/"
    }

    request := "GET " + path + " HTTP/1.1\r\n" + 
                "Host: " + hostname + ":80\r\n" + 
                "User-Agent: Go-Runtime\r\n" + 
                "Accept: */*\r\n" +
                "Accept-Encoding: gzip, deflate, br\r\n" + 
                "Connection: keep-alive\r\n\r\n"

    _, err = fmt.Fprintf(conn, request)
    if err != nil {
        fmt.Println("Error sending request to serrver")
        os.Exit(1)
    }

    r := bufio.NewReader(conn)
    var content_length int
    content_bool := false
    transfer_encoding := false 
    for {
        line, err := r.ReadString('\n')
        
        if err != nil {
            fmt.Println("Error reading header line")
            os.Exit(1)
        }

        header := strings.Split(line, ": ")      
        if header[0] == "Content-Length" {
            content_length, err = strconv.Atoi(header[1][:len(header[1]) - 2])
            if err != nil {
                fmt.Println("Error reading content length")
                os.Exit(1)
            }
            content_bool = true
        }

        if (header[0] == "Transfer-Encoding") {
            transfer_encoding = true
        }
        
        if line == "\r\n" {
            break
        }
    }

    var body []byte
    var n int

    if content_bool == true {
        body = make([]byte, content_length)
        n, err = r.Read(body)
        if err != nil {
            fmt.Println("Error reading body")
            os.Exit(1)
        }
        fmt.Println(string(body[:n]))
    }

    if transfer_encoding == true {
        
        for {
            line, err := r.ReadString('\n')
            if err != nil {
                fmt.Println("Error reading length of chunk")
                os.Exit(1)
            }
            num := line[:len(line) - 2]
            if part_length, err := strconv.ParseInt(num, 16, 64); err == nil {

                if part_length <= 0 {
                    break
                }
                body_part := make([]byte, part_length)
                if _, err := io.ReadFull(r, body_part); err != nil {
                    fmt.Println("Error reading chunk data")
                    os.Exit(1)
                }
                body = append(body, body_part...)
                _, err = r.Discard(2)
                if err != nil {
                    fmt.Println("Failed to discard trailing CRLF of chunk")
                    os.Exit(1)
                }
            } else {
                break
            }
        }

        fmt.Println(body)

        var dat map[string]interface{}
        if err := json.Unmarshal(body, &dat); err != nil {
            panic(err)
        }
        fmt.Println(dat)
        
    }

}

Answer 1

The reason why you get this error is you add request header "Accept-Encoding: gzip, deflate, br\r\n".

So gzip is the first priority for the response encoding.

You can check the response encoding by reading the content-encoding in header.

The first byte '\x1f' is actualy 1 of the 2 magic bytes '0x1f8b' of gzip content.

So to handle the gzip you will need to decompress the repsonse before reading them as plain text.

Here is the sample code:

    // Check if the response is encoded in gzip format
    if resp.Header.Get("Content-Encoding") == "gzip" {
        reader, err := gzip.NewReader(resp.Body)
        if err != nil {
            panic(err)
        }
        defer reader.Close()
        // Read the decompressed response body
        body, err := io.ReadAll(reader)
        if err != nil {
            panic(err)
        }
        // Do something with the response body
        fmt.Println(string(body))
    } else {
        // The response is not gzip encoded, so read it directly
        body, err := io.ReadAll(resp.Body)
        if err != nil {
            panic(err)
        }
        // Do something with the response body
        fmt.Println(string(body))
    }
}

Answer 2

\x1f is the "UNIT SEPARATOR" — a control character from the lower part of the ASCII table.

I dunno how it ended up in the response but I might guess that the endpoint might send you multiple JSON documents in a row — separating them with 0x1F, that is, doing a variation of this.

Another possibility is that you have some error with getting the data. You might show us the code so we could try to make a better guess.