Skip to content

Commit

Permalink
deps: update arrow and parquet (#600)
Browse files Browse the repository at this point in the history
* deps: update pingcap/tidb/parser to latest

This is only used in tests, but not updating this causes future commits to fail
to build.

* deps: update apache/arrow to custom tip

This includes lz4 buffer pooling fixes and removing calls to schema.Fields
which improves allocations.

* deps: update parquet-go to 9010539

This pulls in pooling for comparison buffers in parquet, which reduces
allocations.

* deps: update parquet-go to custom tip

This includes a change to buffer dictionary pages which is still pending
review.
  • Loading branch information
asubiotto authored Nov 29, 2023
1 parent 87c7b97 commit 2f06aa9
Show file tree
Hide file tree
Showing 6 changed files with 115 additions and 109 deletions.
8 changes: 4 additions & 4 deletions dynparquet/nil_chunk.go
Original file line number Diff line number Diff line change
Expand Up @@ -243,15 +243,15 @@ func (p *nilPages) SeekToRow(row int64) error {
// ColumnIndex returns the column index of the column chunk. Since the
// NilColumnChunk is a virtual column chunk only for in-memory purposes, it
// returns nil. Implements the parquet.ColumnChunk interface.
func (c *NilColumnChunk) ColumnIndex() parquet.ColumnIndex {
return nil
func (c *NilColumnChunk) ColumnIndex() (parquet.ColumnIndex, error) {
return nil, nil
}

// OffsetIndex returns the offset index of the column chunk. Since the
// NilColumnChunk is a virtual column chunk only for in-memory purposes, it
// returns nil. Implements the parquet.ColumnChunk interface.
func (c *NilColumnChunk) OffsetIndex() parquet.OffsetIndex {
return nil
func (c *NilColumnChunk) OffsetIndex() (parquet.OffsetIndex, error) {
return nil, nil
}

// BloomFilter returns the bloomfilter of the column chunk. Since the
Expand Down
47 changes: 29 additions & 18 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,24 +10,25 @@ require (
github.com/dgryski/go-metro v0.0.0-20211217172704-adc40b04c140
github.com/dustin/go-humanize v1.0.1
github.com/go-kit/log v0.2.1
github.com/google/uuid v1.3.0
github.com/google/uuid v1.3.1
github.com/oklog/ulid v1.3.1
github.com/parquet-go/parquet-go v0.19.1-0.20231121122118-5df9b59a7d50
github.com/pingcap/tidb/parser v0.0.0-20220921115303-5aab87679fde
github.com/parquet-go/parquet-go v0.19.1-0.20231129084429-9010539a4f7a
github.com/pingcap/tidb/parser v0.0.0-20231013125129-93a834a6bf8d
github.com/polarsignals/wal v0.0.0-20231123092250-5d233119cfc9
github.com/prometheus/client_golang v1.16.0
github.com/stretchr/testify v1.8.4
github.com/thanos-io/objstore v0.0.0-20230713070940-eb01c83b89a4
go.opentelemetry.io/otel v1.16.0
go.opentelemetry.io/otel/trace v1.16.0
go.uber.org/goleak v1.1.12
golang.org/x/exp v0.0.0-20230206171751-46f607a40771
golang.org/x/sync v0.3.0
go.uber.org/goleak v1.2.0
golang.org/x/exp v0.0.0-20231006140011-7918f672742d
golang.org/x/sync v0.4.0
google.golang.org/protobuf v1.31.0
)

require (
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/benbjohnson/clock v1.3.5 // indirect
github.com/benbjohnson/immutable v0.4.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.2.0 // indirect
Expand All @@ -38,19 +39,20 @@ require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/efficientgo/core v1.0.0-rc.2 // indirect
github.com/go-logfmt/logfmt v0.6.0 // indirect
github.com/goccy/go-json v0.10.0 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/flatbuffers v23.1.21+incompatible // indirect
github.com/google/flatbuffers v23.5.26+incompatible // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pierrec/lz4/v4 v4.1.18 // indirect
github.com/pingcap/errors v0.11.5-0.20210425183316-da1aaba5fb63 // indirect
github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7 // indirect
github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect
github.com/pingcap/log v1.1.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
Expand All @@ -60,14 +62,23 @@ require (
github.com/segmentio/encoding v0.3.6 // indirect
github.com/zeebo/xxh3 v1.0.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
go.uber.org/zap v1.18.1 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/text v0.9.0 // indirect
golang.org/x/tools v0.6.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.25.0 // indirect
golang.org/x/mod v0.13.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/tools v0.14.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

// Remove when the following PRs are available in a release.
// - https://github.com/apache/arrow/pull/38729
// - https://github.com/apache/arrow/pull/38919
replace github.com/apache/arrow/go/v14 => github.com/asubiotto/arrow/go/v14 v14.0.0-20231129090719-b321865d34e9

// Remove when the following PRs are available in a release.
// - https://github.com/parquet-go/parquet-go/pull/96
replace github.com/parquet-go/parquet-go => github.com/asubiotto/parquet-go v0.0.0-20231129095010-9bfc4cead953
Loading

0 comments on commit 2f06aa9

Please sign in to comment.