Skip to content

Commit

Permalink
internal/filedesc: implement runtime editions support
Browse files Browse the repository at this point in the history
This change add parsing editions specific descriptor parts to
internal/filedesc which makes it possible to load file descriptors of
protos using editions at runtime.

Change-Id: I7a0c23303acddd2cff115859d4cf82bf0102b14c
Reviewed-on: https://go-review.googlesource.com/c/protobuf/+/563615
Reviewed-by: Michael Stapelberg <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>
  • Loading branch information
lfolger committed Feb 13, 2024
1 parent f387b03 commit 020ec10
Show file tree
Hide file tree
Showing 11 changed files with 415 additions and 11 deletions.
1 change: 1 addition & 0 deletions cmd/protoc-gen-go/testdata/gen_test.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 9 additions & 7 deletions internal/cmd/generate-protos/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ func main() {
}

func generateEditionsDefaults() {
dest := filepath.Join(repoRoot, "reflect", "protodesc", "editions_defaults.binpb")
dest := filepath.Join(repoRoot, "internal", "editiondefaults", "editions_defaults.binpb")
// The enum in Go string formats to "EDITION_${EDITION}" but protoc expects
// the flag in the form "${EDITION}". To work around this, we trim the
// "EDITION_" prefix.
Expand Down Expand Up @@ -185,12 +185,6 @@ func generateLocalProtos() {
if filepath.Base(d.path) == "testdata" {
var imports []string
for sd := range subDirs {
// TODO remove once editions runtime support has landed. Right
// now editions based protos will crash the runtime test during
// init.
if strings.Contains(sd, "editions") {
continue
}
imports = append(imports, fmt.Sprintf("_ %q", path.Join(modulePath, d.path, filepath.ToSlash(sd))))
}
sort.Strings(imports)
Expand Down Expand Up @@ -323,6 +317,14 @@ func generateIdentifiers(gen *protogen.Plugin, file *protogen.File) {
g.P(enum.GoIdent.GoName, "_enum_name = ", strconv.Quote(string(enum.Desc.Name())))
g.P(")")
g.P()

g.P("// Enum values for ", enum.Desc.FullName(), ".")
g.P("const (")
for _, v := range enum.Values {
g.P(v.GoIdent.GoName, "_enum_value = ", v.Desc.Number())
}
g.P(")")
g.P()
}
}
processMessages = func(messages []*protogen.Message) {
Expand Down
12 changes: 12 additions & 0 deletions internal/editiondefaults/defaults.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package editiondefaults contains the binary representation of the editions
// defaults.
package editiondefaults

import _ "embed"

//go:embed editions_defaults.binpb
var Defaults []byte
File renamed without changes.
41 changes: 41 additions & 0 deletions internal/filedesc/desc_init.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
package filedesc

import (
"fmt"
"sync"

"google.golang.org/protobuf/encoding/protowire"
Expand Down Expand Up @@ -98,6 +99,7 @@ func (fd *File) unmarshalSeed(b []byte) {
var prevField protoreflect.FieldNumber
var numEnums, numMessages, numExtensions, numServices int
var posEnums, posMessages, posExtensions, posServices int
var options []byte
b0 := b
for len(b) > 0 {
num, typ, n := protowire.ConsumeTag(b)
Expand All @@ -113,13 +115,17 @@ func (fd *File) unmarshalSeed(b []byte) {
fd.L1.Syntax = protoreflect.Proto2
case "proto3":
fd.L1.Syntax = protoreflect.Proto3
case "editions":
fd.L1.Syntax = protoreflect.Editions
default:
panic("invalid syntax")
}
case genid.FileDescriptorProto_Name_field_number:
fd.L1.Path = sb.MakeString(v)
case genid.FileDescriptorProto_Package_field_number:
fd.L1.Package = protoreflect.FullName(sb.MakeString(v))
case genid.FileDescriptorProto_Options_field_number:
options = v
case genid.FileDescriptorProto_EnumType_field_number:
if prevField != genid.FileDescriptorProto_EnumType_field_number {
if numEnums > 0 {
Expand Down Expand Up @@ -154,6 +160,13 @@ func (fd *File) unmarshalSeed(b []byte) {
numServices++
}
prevField = num
case protowire.VarintType:
v, m := protowire.ConsumeVarint(b)
b = b[m:]
switch num {
case genid.FileDescriptorProto_Edition_field_number:
fd.L1.Edition = Edition(v)
}
default:
m := protowire.ConsumeFieldValue(num, typ, b)
b = b[m:]
Expand All @@ -166,6 +179,11 @@ func (fd *File) unmarshalSeed(b []byte) {
fd.L1.Syntax = protoreflect.Proto2
}

// Parse editions features from options if any
if options != nil {
fd.unmarshalSeedOptions(options)
}

// Must allocate all declarations before parsing each descriptor type
// to ensure we handled all descriptors in "flattened ordering".
if numEnums > 0 {
Expand Down Expand Up @@ -219,6 +237,29 @@ func (fd *File) unmarshalSeed(b []byte) {
}
}

func (fd *File) unmarshalSeedOptions(b []byte) {
for b := b; len(b) > 0; {
num, typ, n := protowire.ConsumeTag(b)
b = b[n:]
switch typ {
case protowire.BytesType:
v, m := protowire.ConsumeBytes(b)
b = b[m:]
switch num {
case genid.FileOptions_Features_field_number:
if fd.Syntax() != protoreflect.Editions {
panic(fmt.Sprintf("invalid descriptor: using edition features in a proto with syntax %s", fd.Syntax()))
}
dfs := getFeaturesFor(fd.L1.Edition)
fd.L1.EditionFeatures = unmarshalFeatureSet(v, dfs)
}
default:
m := protowire.ConsumeFieldValue(num, typ, b)
b = b[m:]
}
}
}

func (ed *Enum) unmarshalSeed(b []byte, sb *strs.Builder, pf *File, pd protoreflect.Descriptor, i int) {
ed.L0.ParentFile = pf
ed.L0.Parent = pd
Expand Down
16 changes: 16 additions & 0 deletions internal/filedesc/desc_lazy.go
Original file line number Diff line number Diff line change
Expand Up @@ -280,6 +280,7 @@ func (md *Message) unmarshalFull(b []byte, sb *strs.Builder) {
var rawFields, rawOneofs [][]byte
var enumIdx, messageIdx, extensionIdx int
var rawOptions []byte
md.L1.EditionFeatures = featuresFromParentDesc(md.Parent())
md.L2 = new(MessageL2)
for len(b) > 0 {
num, typ, n := protowire.ConsumeTag(b)
Expand Down Expand Up @@ -352,6 +353,13 @@ func (md *Message) unmarshalOptions(b []byte) {
case genid.MessageOptions_MessageSetWireFormat_field_number:
md.L1.IsMessageSet = protowire.DecodeBool(v)
}
case protowire.BytesType:
v, m := protowire.ConsumeBytes(b)
b = b[m:]
switch num {
case genid.MessageOptions_Features_field_number:
md.L1.EditionFeatures = unmarshalFeatureSet(v, md.L1.EditionFeatures)
}
default:
m := protowire.ConsumeFieldValue(num, typ, b)
b = b[m:]
Expand Down Expand Up @@ -414,6 +422,7 @@ func (fd *Field) unmarshalFull(b []byte, sb *strs.Builder, pf *File, pd protoref
fd.L0.ParentFile = pf
fd.L0.Parent = pd
fd.L0.Index = i
fd.L1.EditionFeatures = featuresFromParentDesc(fd.Parent())

var rawTypeName []byte
var rawOptions []byte
Expand Down Expand Up @@ -497,6 +506,13 @@ func (fd *Field) unmarshalOptions(b []byte) {
fd.L1.HasEnforceUTF8 = true
fd.L1.EnforceUTF8 = protowire.DecodeBool(v)
}
case protowire.BytesType:
v, m := protowire.ConsumeBytes(b)
b = b[m:]
switch num {
case genid.MessageOptions_Features_field_number:
fd.L1.EditionFeatures = unmarshalFeatureSet(v, fd.L1.EditionFeatures)
}
default:
m := protowire.ConsumeFieldValue(num, typ, b)
b = b[m:]
Expand Down
142 changes: 142 additions & 0 deletions internal/filedesc/editions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package filedesc

import (
"fmt"

"google.golang.org/protobuf/encoding/protowire"
"google.golang.org/protobuf/internal/editiondefaults"
"google.golang.org/protobuf/internal/genid"
"google.golang.org/protobuf/reflect/protoreflect"
)

var defaultsCache = make(map[Edition]EditionFeatures)

func init() {
unmarshalEditionDefaults(editiondefaults.Defaults)
}

func unmarshalGoFeature(b []byte, parent EditionFeatures) EditionFeatures {
for len(b) > 0 {
num, _, n := protowire.ConsumeTag(b)
b = b[n:]
switch num {
case genid.GoFeatures_LegacyUnmarshalJsonEnum_field_number:
v, m := protowire.ConsumeVarint(b)
b = b[m:]
parent.GenerateLegacyUnmarshalJSON = protowire.DecodeBool(v)
default:
panic(fmt.Sprintf("unkown field number %d while unmarshalling GoFeatures", num))
}
}
return parent
}

func unmarshalFeatureSet(b []byte, parent EditionFeatures) EditionFeatures {
for len(b) > 0 {
num, typ, n := protowire.ConsumeTag(b)
b = b[n:]
switch typ {
case protowire.VarintType:
v, m := protowire.ConsumeVarint(b)
b = b[m:]
switch num {
case genid.FeatureSet_FieldPresence_field_number:
parent.IsFieldPresence = v == genid.FeatureSet_EXPLICIT_enum_value || v == genid.FeatureSet_LEGACY_REQUIRED_enum_value
parent.IsLegacyRequired = v == genid.FeatureSet_LEGACY_REQUIRED_enum_value
case genid.FeatureSet_EnumType_field_number:
parent.IsOpenEnum = v == genid.FeatureSet_OPEN_enum_value
case genid.FeatureSet_RepeatedFieldEncoding_field_number:
parent.IsPacked = v == genid.FeatureSet_PACKED_enum_value
case genid.FeatureSet_Utf8Validation_field_number:
parent.IsUTF8Validated = v == genid.FeatureSet_VERIFY_enum_value
case genid.FeatureSet_MessageEncoding_field_number:
parent.IsDelimitedEncoded = v == genid.FeatureSet_DELIMITED_enum_value
case genid.FeatureSet_JsonFormat_field_number:
parent.IsJSONCompliant = v == genid.FeatureSet_ALLOW_enum_value
default:
panic(fmt.Sprintf("unkown field number %d while unmarshalling FeatureSet", num))
}
case protowire.BytesType:
v, m := protowire.ConsumeBytes(b)
b = b[m:]
switch num {
case genid.GoFeatures_LegacyUnmarshalJsonEnum_field_number:
parent = unmarshalGoFeature(v, parent)
}
}
}

return parent
}

func featuresFromParentDesc(parentDesc protoreflect.Descriptor) EditionFeatures {
var parentFS EditionFeatures
switch p := parentDesc.(type) {
case *File:
parentFS = p.L1.EditionFeatures
case *Message:
parentFS = p.L1.EditionFeatures
default:
panic(fmt.Sprintf("unknown parent type %T", parentDesc))
}
return parentFS
}

func unmarshalEditionDefault(b []byte) {
var ed Edition
var fs EditionFeatures
for len(b) > 0 {
num, typ, n := protowire.ConsumeTag(b)
b = b[n:]
switch typ {
case protowire.VarintType:
v, m := protowire.ConsumeVarint(b)
b = b[m:]
switch num {
case genid.FeatureSetDefaults_FeatureSetEditionDefault_Edition_field_number:
ed = Edition(v)
}
case protowire.BytesType:
v, m := protowire.ConsumeBytes(b)
b = b[m:]
switch num {
case genid.FeatureSetDefaults_FeatureSetEditionDefault_Features_field_number:
fs = unmarshalFeatureSet(v, fs)
}
}
}
defaultsCache[ed] = fs
}

func unmarshalEditionDefaults(b []byte) {
for len(b) > 0 {
num, _, n := protowire.ConsumeTag(b)
b = b[n:]
switch num {
case genid.FeatureSetDefaults_Defaults_field_number:
def, m := protowire.ConsumeBytes(b)
b = b[m:]
unmarshalEditionDefault(def)
case genid.FeatureSetDefaults_MinimumEdition_field_number,
genid.FeatureSetDefaults_MaximumEdition_field_number:
// We don't care about the minimum and maximum editions. If the
// edition we are looking for later on is not in the cache we know
// it is outside of the range between minimum and maximum edition.
_, m := protowire.ConsumeVarint(b)
b = b[m:]
default:
panic(fmt.Sprintf("unkown field number %d while unmarshalling EditionDefault", num))
}
}
}

func getFeaturesFor(ed Edition) EditionFeatures {
if def, ok := defaultsCache[ed]; ok {
return def
}
panic(fmt.Sprintf("unsupported edition: %v", ed))
}
Loading

0 comments on commit 020ec10

Please sign in to comment.