Convert protobuf serialized messages to JSON without precompiling Go code

Question

I want to convert protobuf serialized messages into a human readable JSON format. The major problem I face is that I need to do this without compiling the proto descriptor into Go code beforehand. I have access to the .proto files at runtime, but not at compile time.

I had the impression that the new Protobuf API v2 (https://github.com/protocolbuffers/protobuf-go) supports dynamic deserialization (see package types/dynamicpb), but I couldn't figure out how to use it apparently:

func readDynamically(in []byte) {
    // How do I load the required descriptor (for NewMessage()) from my `addressbook.proto` file?)
    descriptor := ??

    msg := dynamicpb.NewMessage(descriptor)
    err := protojson.Unmarshal(in, msg)
    if err != nil {
        panic(err)
    }
}

Above code is annotated with my problem: How can I get the required descriptor for the dynamicpb.NewMessage() from a .proto file?

Answer 1

Should work like this with the dynamicpb package.

func readDynamically(in []byte) {
    registry, err := createProtoRegistry(".", "addressbook.proto")
    if err != nil {
        panic(err)
    }

    desc, err := registry.FindFileByPath("addressbook.proto")
    if err != nil {
        panic(err)
    }
    fd := desc.Messages()
    addressBook := fd.ByName("AddressBook")

    msg := dynamicpb.NewMessage(addressBook)
    err = proto.Unmarshal(in, msg)
    jsonBytes, err := protojson.Marshal(msg)
    if err != nil {
        panic(err)
    }
    fmt.Println(string(jsonBytes))
    if err != nil {
        panic(err)
    }
}

func createProtoRegistry(srcDir string, filename string) (*protoregistry.Files, error) {
    // Create descriptors using the protoc binary.
    // Imported dependencies are included so that the descriptors are self-contained.
    tmpFile := filename + "-tmp.pb"
    cmd := exec.Command("./protoc/protoc",
        "--include_imports",
        "--descriptor_set_out=" + tmpFile,
        "-I"+srcDir,
        path.Join(srcDir, filename))

    cmd.Stdout = os.Stdout
    cmd.Stderr = os.Stderr
    err := cmd.Run()
    if err != nil {
        return nil, err
    }
    defer os.Remove(tmpFile)

    marshalledDescriptorSet, err := ioutil.ReadFile(tmpFile)
    if err != nil {
        return nil, err
    }
    descriptorSet := descriptorpb.FileDescriptorSet{}
    err = proto.Unmarshal(marshalledDescriptorSet, &descriptorSet)
    if err != nil {
        return nil, err
    }

    files, err := protodesc.NewFiles(&descriptorSet)
    if err != nil {
        return nil, err
    }

    return files, nil
}

Answer 2

This question is kind of interesting. I have done some works on protobuf plugs. As far as i can tell, additional cli is needed because we don't want to "reinvent the wheel".

Step one, we need protoc to translate ".proto" file to some format so we can get "protoreflect.MessageDescriptor" easily.

This plug is to get raw bytes which protoc sends to other plugs as input.

package main

import (
    "fmt"
    "io/ioutil"
    "os"
)

func main() {
    if len(os.Args) == 2 && os.Args[1] == "--version" {
        // fmt.Fprintf(os.Stderr, "%v %v\n", filepath.Base(os.Args[0]), version.String())
        os.Exit(0)
    }

    in, err := ioutil.ReadAll(os.Stdin)
    if err != nil {
        fmt.Printf("error: %v", err)
        return
    }
    ioutil.WriteFile("./out.pb", in, 0755)
}

build and rename it as protoc-gen-raw, then generate protoc --raw_out=./pb ./server.proto, you will get out.pb. Forget your ".proto" file from now on, and put this "out.pb" where you intend to put ".proto". And what we get is official support with this .pb file.

Step 2: Deserialize a protobuf serialized message into JSON.

package main

import (
    "fmt"
    "io/ioutil"

    "google.golang.org/protobuf/proto"
    "google.golang.org/protobuf/compiler/protogen"
    "google.golang.org/protobuf/encoding/protojson"
    "google.golang.org/protobuf/types/dynamicpb"
    "google.golang.org/protobuf/types/pluginpb"
)

func main() {
    in, err := ioutil.ReadFile("./out.pb")
    if err != nil {
        fmt.Printf("failed to read proto file: %v", err)
        return
    }
    req := &pluginpb.CodeGeneratorRequest{}
    if err := proto.Unmarshal(in, req); err != nil {
        fmt.Printf("failed to unmarshal proto: %v", err)
        return
    }
    gen, err := protogen.Options{}.New(req)
    if err != nil {
        fmt.Printf("failed to create new plugin: %v", err)
        return
    }

    // serialize protobuf message "ServerConfig"
    data := &ServerConfig{
        GameType: 1,
        ServerId: 105,
        Host:     "host.host.host",
        Port:     10024,
    }
    raw, err := data.Marshal()
    if err != nil {
        fmt.Printf("failed to marshal protobuf: %v", err)
        return
    }

    for _, f := range gen.Files {
        for _, m := range f.Messages {

            // "ServerConfig" is the message name of the serialized message
            if m.GoIdent.GoName == "ServerConfig" {
                // m.Desc is MessageDescriptor
                msg := dynamicpb.NewMessage(m.Desc)
                // unmarshal []byte into proto message
                err := proto.Unmarshal(raw, msg)
                if err != nil {
                    fmt.Printf("failed to Unmarshal protobuf data: %v", err)
                    return
                }
                // marshal message into json
                jsondata, err := protojson.Marshal(msg)
                if err != nil {
                    fmt.Printf("failed to Marshal to json: %v", err)
                    return
                }
                fmt.Printf("out: %v", string(jsondata))
            }
        }
    }
}
// the output is:
// out: {"gameType":1, "serverId":105, "host":"host.host.host", "port":10024}