diff --git a/.gitignore b/.gitignore index 435194ee..b2d23a77 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ fq-connector-go +coverage.out diff --git a/Makefile b/Makefile index 111137b4..83135501 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,9 @@ build: go build -o fq-connector-go ./app +run: build + ./fq-connector-go server -c ./example.conf + unit_test: go test -v ./app/... diff --git a/app/client/client.go b/app/client/client.go index 177e9f00..1bf34ed8 100644 --- a/app/client/client.go +++ b/app/client/client.go @@ -17,7 +17,7 @@ import ( api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" "github.com/ydb-platform/fq-connector-go/app/config" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" @@ -64,7 +64,7 @@ func runClient(_ *cobra.Command, args []string) error { return nil } -func makeConnection(logger log.Logger, cfg *config.ClientConfig) (*grpc.ClientConn, error) { +func makeConnection(logger *zap.Logger, cfg *config.ClientConfig) (*grpc.ClientConn, error) { var opts []grpc.DialOption if cfg.Tls != nil { @@ -99,7 +99,7 @@ func makeConnection(logger log.Logger, cfg *config.ClientConfig) (*grpc.ClientCo return conn, nil } -func callServer(logger log.Logger, cfg *config.ClientConfig) error { +func callServer(logger *zap.Logger, cfg *config.ClientConfig) error { conn, err := makeConnection(logger, cfg) if err != nil { return fmt.Errorf("grpc dial: %w", err) @@ -140,12 +140,12 @@ func callServer(logger log.Logger, cfg *config.ClientConfig) error { } func describeTable( - logger log.Logger, + logger *zap.Logger, connectorClient api_service.ConnectorClient, dsi *api_common.TDataSourceInstance, ) (*api_service_protos.TSchema, error) { req := &api_service_protos.TDescribeTableRequest{Table: tableName, DataSourceInstance: dsi} - logger.Debug("DescribeTable", log.String("request", req.String())) + logger.Debug("DescribeTable", zap.String("request", req.String())) resp, err := connectorClient.DescribeTable(context.TODO(), req) if err != nil { @@ -153,18 +153,18 @@ func describeTable( } if utils.IsSuccess(resp.Error) { - logger.Debug("DescribeTable", log.String("response", resp.String())) + logger.Debug("DescribeTable", zap.String("response", resp.String())) return resp.Schema, nil } - logger.Error("DescribeTable", log.String("response", resp.String())) + logger.Error("DescribeTable", zap.String("response", resp.String())) return nil, utils.NewSTDErrorFromAPIError(resp.Error) } func listSplits( - logger log.Logger, + logger *zap.Logger, schema *api_service_protos.TSchema, connectorClient api_service.ConnectorClient, dsi *api_common.TDataSourceInstance, @@ -186,7 +186,7 @@ func listSplits( }, }, } - logger.Debug("ListSplits", log.String("request", req.String())) + logger.Debug("ListSplits", zap.String("request", req.String())) streamListSplits, err := connectorClient.ListSplits(context.TODO(), req) if err != nil { @@ -206,12 +206,12 @@ func listSplits( } if !utils.IsSuccess(resp.Error) { - logger.Error("ListSplits", log.String("response", resp.String())) + logger.Error("ListSplits", zap.String("response", resp.String())) return splits, utils.NewSTDErrorFromAPIError(resp.Error) } - logger.Debug("ListSplits", log.String("response", resp.String())) + logger.Debug("ListSplits", zap.String("response", resp.String())) splits = append(splits, resp.Splits...) } @@ -223,14 +223,14 @@ func listSplits( } func readSplits( - logger log.Logger, + logger *zap.Logger, splits []*api_service_protos.TSplit, format api_service_protos.TReadSplitsRequest_EFormat, connectorClient api_service.ConnectorClient, dsi *api_common.TDataSourceInstance, ) error { req := &api_service_protos.TReadSplitsRequest{Splits: splits, Format: format, DataSourceInstance: dsi} - logger.Debug("ReadSplits", log.String("request", req.String())) + logger.Debug("ReadSplits", zap.String("request", req.String())) streamReadSplits, err := connectorClient.ReadSplits(context.Background(), req) if err != nil { @@ -264,7 +264,7 @@ func readSplits( } func dumpReadResponses( - logger log.Logger, + logger *zap.Logger, format api_service_protos.TReadSplitsRequest_EFormat, responses []*api_service_protos.TReadSplitsResponse, ) error { @@ -279,10 +279,10 @@ func dumpReadResponses( for reader.Next() { record := reader.Record() - logger.Debug("schema", log.String("schema", record.Schema().String())) + logger.Debug("schema", zap.String("schema", record.Schema().String())) for i, column := range record.Columns() { - logger.Debug("column", log.Int("id", i), log.String("data", column.String())) + logger.Debug("column", zap.Int("id", i), zap.String("data", column.String())) } } diff --git a/app/server/cmd.go b/app/server/cmd.go index a5631ee0..27e2dbcc 100644 --- a/app/server/cmd.go +++ b/app/server/cmd.go @@ -2,7 +2,6 @@ package server import ( "fmt" - "log" "os" "github.com/spf13/cobra" @@ -25,6 +24,7 @@ func init() { Cmd.Flags().StringP(configFlag, "c", "", "path to server config file") if err := Cmd.MarkFlagRequired(configFlag); err != nil { - log.Fatal(err) + fmt.Println(err) + os.Exit(1) } } diff --git a/app/server/data_source_collection.go b/app/server/data_source_collection.go index a3aca93f..a7d666a1 100644 --- a/app/server/data_source_collection.go +++ b/app/server/data_source_collection.go @@ -15,7 +15,7 @@ import ( "github.com/ydb-platform/fq-connector-go/app/server/paging" "github.com/ydb-platform/fq-connector-go/app/server/streaming" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) type DataSourceCollection struct { @@ -26,7 +26,7 @@ type DataSourceCollection struct { } func (dsc *DataSourceCollection) DescribeTable( - ctx context.Context, logger log.Logger, request *api_service_protos.TDescribeTableRequest, + ctx context.Context, logger *zap.Logger, request *api_service_protos.TDescribeTableRequest, ) (*api_service_protos.TDescribeTableResponse, error) { kind := request.GetDataSourceInstance().GetKind() @@ -48,7 +48,7 @@ func (dsc *DataSourceCollection) DescribeTable( } func (dsc *DataSourceCollection) DoReadSplit( - logger log.Logger, + logger *zap.Logger, stream api_service.Connector_ReadSplitsServer, request *api_service_protos.TReadSplitsRequest, split *api_service_protos.TSplit, @@ -71,7 +71,7 @@ func (dsc *DataSourceCollection) DoReadSplit( } func readSplit[T utils.Acceptor]( - logger log.Logger, + logger *zap.Logger, stream api_service.Connector_ReadSplitsServer, request *api_service_protos.TReadSplitsRequest, split *api_service_protos.TSplit, @@ -122,8 +122,8 @@ func readSplit[T utils.Acceptor]( logger.Debug( "split reading finished", - log.UInt64("total_bytes", readStats.GetBytes()), - log.UInt64("total_rows", readStats.GetRows()), + zap.Uint64("total_bytes", readStats.GetBytes()), + zap.Uint64("total_rows", readStats.GetRows()), ) return nil diff --git a/app/server/datasource/interface.go b/app/server/datasource/interface.go index 9c55e3b8..a6883730 100644 --- a/app/server/datasource/interface.go +++ b/app/server/datasource/interface.go @@ -7,12 +7,12 @@ import ( api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" "github.com/ydb-platform/fq-connector-go/app/server/paging" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) type DataSourceFactory[T utils.Acceptor] interface { Make( - logger log.Logger, + logger *zap.Logger, dataSourceType api_common.EDataSourceKind, ) (DataSource[T], error) } @@ -25,14 +25,14 @@ type DataSource[T utils.Acceptor] interface { // located within a particular database in a data source cluster. DescribeTable( ctx context.Context, - logger log.Logger, + logger *zap.Logger, request *api_service_protos.TDescribeTableRequest, ) (*api_service_protos.TDescribeTableResponse, error) // ReadSplit is a main method for reading data from the table. ReadSplit( ctx context.Context, - logger log.Logger, + logger *zap.Logger, split *api_service_protos.TSplit, sink paging.Sink[T], ) diff --git a/app/server/datasource/mock.go b/app/server/datasource/mock.go index 05918291..96dc8485 100644 --- a/app/server/datasource/mock.go +++ b/app/server/datasource/mock.go @@ -7,7 +7,7 @@ import ( api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" "github.com/ydb-platform/fq-connector-go/app/server/paging" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) var _ DataSource[any] = (*DataSourceMock[any])(nil) @@ -18,7 +18,7 @@ type DataSourceMock[T utils.Acceptor] struct { func (m *DataSourceMock[T]) DescribeTable( _ context.Context, - _ log.Logger, + _ *zap.Logger, _ *api_service_protos.TDescribeTableRequest, ) (*api_service_protos.TDescribeTableResponse, error) { panic("not implemented") // TODO: Implement @@ -26,7 +26,7 @@ func (m *DataSourceMock[T]) DescribeTable( func (m *DataSourceMock[T]) ReadSplit( _ context.Context, - _ log.Logger, + _ *zap.Logger, split *api_service_protos.TSplit, pagingWriter paging.Sink[T], ) { diff --git a/app/server/datasource/rdbms/clickhouse/connection_manager.go b/app/server/datasource/rdbms/clickhouse/connection_manager.go index 362f6dc7..fd8dbf74 100644 --- a/app/server/datasource/rdbms/clickhouse/connection_manager.go +++ b/app/server/datasource/rdbms/clickhouse/connection_manager.go @@ -11,8 +11,8 @@ import ( api_common "github.com/ydb-platform/fq-connector-go/api/common" rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" + "go.uber.org/zap" ) var _ rdbms_utils.Connection = (*Connection)(nil) @@ -56,7 +56,7 @@ func (c Connection) Query(ctx context.Context, query string, args ...any) (rdbms if err := out.Err(); err != nil { defer func() { if closeErr := out.Close(); closeErr != nil { - c.logger.Error("close rows", log.Error(closeErr)) + c.logger.Error("close rows", zap.Error(closeErr)) } }() @@ -75,7 +75,7 @@ type connectionManager struct { func (c *connectionManager) Make( _ context.Context, - logger log.Logger, + logger *zap.Logger, dsi *api_common.TDataSourceInstance, ) (rdbms_utils.Connection, error) { if dsi.GetCredentials().GetBasic() == nil { @@ -103,7 +103,7 @@ func (c *connectionManager) Make( // Set this field to true if you want to see ClickHouse driver's debug output Debug: false, Debugf: func(format string, v ...any) { - logger.Debugf(format, v...) + logger.Debug(format, zap.Any("args", v)) }, // TODO: make it configurable via Connector API Compression: &clickhouse.Compression{ @@ -138,7 +138,7 @@ func (c *connectionManager) Make( return &Connection{DB: conn, logger: queryLogger}, nil } -func (c *connectionManager) Release(logger log.Logger, conn rdbms_utils.Connection) { +func (c *connectionManager) Release(logger *zap.Logger, conn rdbms_utils.Connection) { utils.LogCloserError(logger, conn, "close clickhouse connection") } diff --git a/app/server/datasource/rdbms/data_source.go b/app/server/datasource/rdbms/data_source.go index ebd4055c..a3d80b0a 100644 --- a/app/server/datasource/rdbms/data_source.go +++ b/app/server/datasource/rdbms/data_source.go @@ -9,7 +9,7 @@ import ( rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" "github.com/ydb-platform/fq-connector-go/app/server/paging" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) type Preset struct { @@ -26,12 +26,12 @@ type dataSourceImpl struct { sqlFormatter rdbms_utils.SQLFormatter connectionManager rdbms_utils.ConnectionManager schemaProvider rdbms_utils.SchemaProvider - logger log.Logger + logger *zap.Logger } func (ds *dataSourceImpl) DescribeTable( ctx context.Context, - logger log.Logger, + logger *zap.Logger, request *api_service_protos.TDescribeTableRequest, ) (*api_service_protos.TDescribeTableResponse, error) { conn, err := ds.connectionManager.Make(ctx, logger, request.DataSourceInstance) @@ -51,7 +51,7 @@ func (ds *dataSourceImpl) DescribeTable( func (ds *dataSourceImpl) doReadSplit( ctx context.Context, - logger log.Logger, + logger *zap.Logger, split *api_service_protos.TSplit, sink paging.Sink[any], ) error { @@ -105,7 +105,7 @@ func (ds *dataSourceImpl) doReadSplit( func (ds *dataSourceImpl) ReadSplit( ctx context.Context, - logger log.Logger, + logger *zap.Logger, split *api_service_protos.TSplit, sink paging.Sink[any], ) { @@ -118,7 +118,7 @@ func (ds *dataSourceImpl) ReadSplit( } func NewDataSource( - logger log.Logger, + logger *zap.Logger, preset *Preset, ) datasource.DataSource[any] { return &dataSourceImpl{ diff --git a/app/server/datasource/rdbms/data_source_factory.go b/app/server/datasource/rdbms/data_source_factory.go index 7906a496..a51911ff 100644 --- a/app/server/datasource/rdbms/data_source_factory.go +++ b/app/server/datasource/rdbms/data_source_factory.go @@ -10,7 +10,7 @@ import ( rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/ydb" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) var _ datasource.DataSourceFactory[any] = (*dataSourceFactory)(nil) @@ -22,7 +22,7 @@ type dataSourceFactory struct { } func (dsf *dataSourceFactory) Make( - logger log.Logger, + logger *zap.Logger, dataSourceType api_common.EDataSourceKind, ) (datasource.DataSource[any], error) { switch dataSourceType { diff --git a/app/server/datasource/rdbms/postgresql/connection_manager.go b/app/server/datasource/rdbms/postgresql/connection_manager.go index 7ee3dbff..7c54540e 100644 --- a/app/server/datasource/rdbms/postgresql/connection_manager.go +++ b/app/server/datasource/rdbms/postgresql/connection_manager.go @@ -10,8 +10,8 @@ import ( api_common "github.com/ydb-platform/fq-connector-go/api/common" rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" + "go.uber.org/zap" ) var _ rdbms_utils.Connection = (*Connection)(nil) @@ -63,7 +63,7 @@ type connectionManager struct { func (c *connectionManager) Make( ctx context.Context, - logger log.Logger, + logger *zap.Logger, dsi *api_common.TDataSourceInstance, ) (rdbms_utils.Connection, error) { if dsi.GetCredentials().GetBasic() == nil { @@ -116,7 +116,7 @@ func (c *connectionManager) Make( return &Connection{conn, queryLogger}, nil } -func (c *connectionManager) Release(logger log.Logger, conn rdbms_utils.Connection) { +func (c *connectionManager) Release(logger *zap.Logger, conn rdbms_utils.Connection) { utils.LogCloserError(logger, conn, "close posgresql connection") } diff --git a/app/server/datasource/rdbms/utils/default_schema_provider.go b/app/server/datasource/rdbms/utils/default_schema_provider.go index 92c00379..720697aa 100644 --- a/app/server/datasource/rdbms/utils/default_schema_provider.go +++ b/app/server/datasource/rdbms/utils/default_schema_provider.go @@ -6,7 +6,7 @@ import ( api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" "github.com/ydb-platform/fq-connector-go/app/server/utils" - my_log "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) type DefaultSchemaProvider struct { @@ -18,7 +18,7 @@ var _ SchemaProvider = (*DefaultSchemaProvider)(nil) func (f *DefaultSchemaProvider) GetSchema( ctx context.Context, - logger my_log.Logger, + logger *zap.Logger, conn Connection, request *api_service_protos.TDescribeTableRequest, ) (*api_service_protos.TSchema, error) { diff --git a/app/server/datasource/rdbms/utils/query_builder.go b/app/server/datasource/rdbms/utils/query_builder.go index fec6a02e..beb615c2 100644 --- a/app/server/datasource/rdbms/utils/query_builder.go +++ b/app/server/datasource/rdbms/utils/query_builder.go @@ -5,10 +5,10 @@ import ( "strings" api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) -func MakeReadSplitQuery(logger log.Logger, formatter SQLFormatter, request *api_service_protos.TSelect) (string, []any, error) { +func MakeReadSplitQuery(logger *zap.Logger, formatter SQLFormatter, request *api_service_protos.TSelect) (string, []any, error) { var ( sb strings.Builder args []any @@ -26,7 +26,7 @@ func MakeReadSplitQuery(logger log.Logger, formatter SQLFormatter, request *api_ clause, args, err = formatWhereClause(formatter, request.Where) if err != nil { - logger.Error("Failed to format WHERE clause", log.Error(err), log.String("where", request.Where.String())) + logger.Error("Failed to format WHERE clause", zap.Error(err), zap.String("where", request.Where.String())) } else { sb.WriteString(" ") sb.WriteString(clause) diff --git a/app/server/datasource/rdbms/utils/schema_builder.go b/app/server/datasource/rdbms/utils/schema_builder.go index e99fa521..f360f337 100644 --- a/app/server/datasource/rdbms/utils/schema_builder.go +++ b/app/server/datasource/rdbms/utils/schema_builder.go @@ -6,8 +6,8 @@ import ( api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" + "go.uber.org/zap" ) type schemaItem struct { @@ -40,7 +40,7 @@ func (sb *SchemaBuilder) AddColumn(columnName, columnType string) error { return nil } -func (sb *SchemaBuilder) Build(logger log.Logger) (*api_service_protos.TSchema, error) { +func (sb *SchemaBuilder) Build(logger *zap.Logger) (*api_service_protos.TSchema, error) { if len(sb.items) == 0 { return nil, utils.ErrTableDoesNotExist } @@ -61,7 +61,7 @@ func (sb *SchemaBuilder) Build(logger log.Logger) (*api_service_protos.TSchema, if len(unsupported) > 0 { logger.Warn( "the table schema was reduced because some column types are unsupported", - log.Strings("unsupported columns", unsupported), + zap.Strings("unsupported columns", unsupported), ) } diff --git a/app/server/datasource/rdbms/utils/sql.go b/app/server/datasource/rdbms/utils/sql.go index 3338f008..9a5876f3 100644 --- a/app/server/datasource/rdbms/utils/sql.go +++ b/app/server/datasource/rdbms/utils/sql.go @@ -6,8 +6,8 @@ import ( api_common "github.com/ydb-platform/fq-connector-go/api/common" api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" + "go.uber.org/zap" ) type Connection interface { @@ -24,8 +24,8 @@ type Rows interface { } type ConnectionManager interface { - Make(ctx context.Context, logger log.Logger, dataSourceInstance *api_common.TDataSourceInstance) (Connection, error) - Release(logger log.Logger, connection Connection) + Make(ctx context.Context, logger *zap.Logger, dataSourceInstance *api_common.TDataSourceInstance) (Connection, error) + Release(logger *zap.Logger, connection Connection) } type ConnectionManagerBase struct { @@ -46,7 +46,7 @@ type SQLFormatter interface { type SchemaProvider interface { GetSchema( ctx context.Context, - logger log.Logger, + logger *zap.Logger, conn Connection, request *api_service_protos.TDescribeTableRequest, ) (*api_service_protos.TSchema, error) diff --git a/app/server/datasource/rdbms/utils/sql_mock.go b/app/server/datasource/rdbms/utils/sql_mock.go index d55991e6..4723af6a 100644 --- a/app/server/datasource/rdbms/utils/sql_mock.go +++ b/app/server/datasource/rdbms/utils/sql_mock.go @@ -7,8 +7,8 @@ import ( "github.com/stretchr/testify/mock" api_common "github.com/ydb-platform/fq-connector-go/api/common" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" + "go.uber.org/zap" ) var _ Connection = (*ConnectionMock)(nil) @@ -35,14 +35,14 @@ type ConnectionManagerMock struct { func (m *ConnectionManagerMock) Make( _ context.Context, - _ log.Logger, + _ *zap.Logger, dataSourceInstance *api_common.TDataSourceInstance) (Connection, error) { args := m.Called(dataSourceInstance) return args.Get(0).(Connection), args.Error(1) } -func (m *ConnectionManagerMock) Release(_ log.Logger, conn Connection) { +func (m *ConnectionManagerMock) Release(_ *zap.Logger, conn Connection) { m.Called(conn) } diff --git a/app/server/datasource/rdbms/ydb/connection_manager.go b/app/server/datasource/rdbms/ydb/connection_manager.go index 6d8e3a1a..ee3cb522 100644 --- a/app/server/datasource/rdbms/ydb/connection_manager.go +++ b/app/server/datasource/rdbms/ydb/connection_manager.go @@ -9,10 +9,10 @@ import ( api_common "github.com/ydb-platform/fq-connector-go/api/common" rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" ydb_sdk "github.com/ydb-platform/ydb-go-sdk/v3" "github.com/ydb-platform/ydb-go-sdk/v3/sugar" + "go.uber.org/zap" ) var _ rdbms_utils.Connection = (*Connection)(nil) @@ -56,7 +56,7 @@ func (c Connection) Query(ctx context.Context, query string, args ...any) (rdbms if err := out.Err(); err != nil { defer func() { if err = out.Close(); err != nil { - c.logger.Error("close rows", log.Error(err)) + c.logger.Error("close rows", zap.Error(err)) } }() @@ -75,7 +75,7 @@ type connectionManager struct { func (c *connectionManager) Make( ctx context.Context, - logger log.Logger, + logger *zap.Logger, dsi *api_common.TDataSourceInstance, ) (rdbms_utils.Connection, error) { // TODO: add credentials (iam and basic) support @@ -109,7 +109,7 @@ func (c *connectionManager) Make( return &Connection{DB: conn, logger: queryLogger}, nil } -func (c *connectionManager) Release(logger log.Logger, conn rdbms_utils.Connection) { +func (c *connectionManager) Release(logger *zap.Logger, conn rdbms_utils.Connection) { utils.LogCloserError(logger, conn, "close clickhouse connection") } diff --git a/app/server/datasource/rdbms/ydb/schema_provider.go b/app/server/datasource/rdbms/ydb/schema_provider.go index 4011ab9a..dbc51d37 100644 --- a/app/server/datasource/rdbms/ydb/schema_provider.go +++ b/app/server/datasource/rdbms/ydb/schema_provider.go @@ -8,10 +8,10 @@ import ( api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" "github.com/ydb-platform/fq-connector-go/app/server/utils" - my_log "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/ydb-go-sdk/v3" "github.com/ydb-platform/ydb-go-sdk/v3/table" "github.com/ydb-platform/ydb-go-sdk/v3/table/options" + "go.uber.org/zap" ) type schemaProvider struct { @@ -22,7 +22,7 @@ var _ rdbms_utils.SchemaProvider = (*schemaProvider)(nil) func (f *schemaProvider) GetSchema( ctx context.Context, - logger my_log.Logger, + logger *zap.Logger, conn rdbms_utils.Connection, request *api_service_protos.TDescribeTableRequest, ) (*api_service_protos.TSchema, error) { diff --git a/app/server/datasource/s3/data_source.go b/app/server/datasource/s3/data_source.go index 1b1da154..8a9b7688 100644 --- a/app/server/datasource/s3/data_source.go +++ b/app/server/datasource/s3/data_source.go @@ -15,8 +15,8 @@ import ( "github.com/ydb-platform/fq-connector-go/app/server/datasource" "github.com/ydb-platform/fq-connector-go/app/server/paging" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" + "go.uber.org/zap" ) var _ datasource.DataSource[string] = (*dataSource)(nil) @@ -26,13 +26,13 @@ type dataSource struct { func (ds *dataSource) DescribeTable( _ context.Context, - _ log.Logger, + _ *zap.Logger, _ *api_service_protos.TDescribeTableRequest, ) (*api_service_protos.TDescribeTableResponse, error) { return nil, fmt.Errorf("table description is not implemented for schemaless data sources: %w", utils.ErrMethodNotSupported) } -func (ds *dataSource) ReadSplit(ctx context.Context, logger log.Logger, split *api_service_protos.TSplit, sink paging.Sink[string]) { +func (ds *dataSource) ReadSplit(ctx context.Context, logger *zap.Logger, split *api_service_protos.TSplit, sink paging.Sink[string]) { if err := ds.doReadSplit(ctx, logger, split, sink); err != nil { sink.AddError(err) } @@ -42,7 +42,7 @@ func (ds *dataSource) ReadSplit(ctx context.Context, logger log.Logger, split *a func (ds *dataSource) doReadSplit( ctx context.Context, - _ log.Logger, + _ *zap.Logger, split *api_service_protos.TSplit, sink paging.Sink[string]) error { conn := makeConnection() diff --git a/app/server/httppuller.go b/app/server/httppuller.go index a1ff9b43..ba7b8e19 100644 --- a/app/server/httppuller.go +++ b/app/server/httppuller.go @@ -6,10 +6,9 @@ import ( "io" "net/http" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "github.com/ydb-platform/fq-connector-go/library/go/core/log/nop" "github.com/ydb-platform/fq-connector-go/library/go/core/metrics/solomon" "github.com/ydb-platform/fq-connector-go/library/go/httputil/headers" + "go.uber.org/zap" ) type MetricsStreamer interface { @@ -20,7 +19,7 @@ type MetricsStreamer interface { type handler struct { registry MetricsStreamer streamFormat headers.ContentType - logger log.Logger + logger *zap.Logger } type spackOption struct { @@ -37,11 +36,11 @@ type Option interface { } // NewHTTPPullerHandler returns new HTTP handler to expose gathered metrics using metrics dumper -func NewHTTPPullerHandler(r MetricsStreamer, opts ...Option) http.Handler { +func NewHTTPPullerHandler(logger *zap.Logger, r MetricsStreamer, opts ...Option) http.Handler { h := handler{ registry: r, streamFormat: headers.TypeApplicationJSON, - logger: &nop.Logger{}, + logger: logger, } for _, opt := range opts { @@ -64,7 +63,7 @@ func (h handler) okSpack(header http.Header) bool { for _, header := range header[headers.AcceptKey] { types, err := headers.ParseAccept(header) if err != nil { - h.logger.Warn("Can't parse accept header", log.Error(err), log.String("header", header)) + h.logger.Warn("Can't parse accept header", zap.Error(err), zap.String("header", header)) continue } @@ -84,7 +83,7 @@ func (h handler) okLZ4Compression(header http.Header) bool { encodings, err := headers.ParseAcceptEncoding(header) if err != nil { - h.logger.Warn("Can't parse accept-encoding header", log.Error(err), log.String("header", header)) + h.logger.Warn("Can't parse accept-encoding header", zap.Error(err), zap.String("header", header)) continue } @@ -111,7 +110,7 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, err := h.registry.StreamSpack(r.Context(), w, compression) if err != nil { - h.logger.Error("Failed to write compressed spack", log.Error(err)) + h.logger.Error("Failed to write compressed spack", zap.Error(err)) } return @@ -121,6 +120,6 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { _, err := h.registry.StreamJSON(r.Context(), w) if err != nil { - h.logger.Error("Failed to write json", log.Error(err)) + h.logger.Error("Failed to write json", zap.Error(err)) } } diff --git a/app/server/launcher.go b/app/server/launcher.go index b987520c..1e9b9a37 100644 --- a/app/server/launcher.go +++ b/app/server/launcher.go @@ -9,8 +9,8 @@ import ( "github.com/spf13/cobra" "github.com/ydb-platform/fq-connector-go/app/config" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/fq-connector-go/library/go/core/metrics/solomon" + "go.uber.org/zap" ) type service interface { @@ -20,7 +20,7 @@ type service interface { type launcher struct { services map[string]service - logger log.Logger + logger *zap.Logger } func (l *launcher) start() <-chan error { @@ -29,7 +29,7 @@ func (l *launcher) start() <-chan error { for key := range l.services { key := key go func(key string) { - l.logger.Info("starting service", log.String("service", key)) + l.logger.Info("starting service", zap.String("service", key)) // blocking call errChan <- l.services[key].start() @@ -42,7 +42,7 @@ func (l *launcher) start() <-chan error { func (l *launcher) stop() { // TODO: make it concurrent for key, s := range l.services { - l.logger.Info("stopping service", log.String("service", key)) + l.logger.Info("stopping service", zap.String("service", key)) s.stop() } } @@ -53,7 +53,7 @@ const ( metricsKey = "metrics" ) -func newLauncher(logger log.Logger, cfg *config.TServerConfig) (*launcher, error) { +func newLauncher(logger *zap.Logger, cfg *config.TServerConfig) (*launcher, error) { l := &launcher{ services: make(map[string]service, 2), logger: logger, @@ -68,13 +68,13 @@ func newLauncher(logger log.Logger, cfg *config.TServerConfig) (*launcher, error if cfg.MetricsServer != nil { l.services[metricsKey] = newServiceMetrics( - log.With(logger, log.String("service", metricsKey)), + logger.With(zap.String("service", metricsKey)), cfg.MetricsServer, registry) } // init GRPC server l.services[connectorServiceKey], err = newServiceConnector( - log.With(logger, log.String("service", connectorServiceKey)), + logger.With(zap.String("service", connectorServiceKey)), cfg, registry) if err != nil { return nil, fmt.Errorf("new connector server: %w", err) @@ -83,7 +83,7 @@ func newLauncher(logger log.Logger, cfg *config.TServerConfig) (*launcher, error // init Pprof server if cfg.PprofServer != nil { l.services[pprofServiceKey] = newServicePprof( - log.With(logger, log.String("service", pprofServiceKey)), + logger.With(zap.String("service", pprofServiceKey)), cfg.PprofServer) } @@ -118,9 +118,9 @@ func run(cmd *cobra.Command, _ []string) error { select { case err := <-errChan: - logger.Error("service fatal error", log.Error(err)) + logger.Error("service fatal error", zap.Error(err)) case sig := <-signalChan: - logger.Info("interrupting signal", log.Any("value", sig)) + logger.Info("interrupting signal", zap.Any("value", sig)) l.stop() } diff --git a/app/server/paging/columnar_buffer_arrow_ipc_streaming_default.go b/app/server/paging/columnar_buffer_arrow_ipc_streaming_default.go index ce8c68cc..fa6c79d9 100644 --- a/app/server/paging/columnar_buffer_arrow_ipc_streaming_default.go +++ b/app/server/paging/columnar_buffer_arrow_ipc_streaming_default.go @@ -10,7 +10,7 @@ import ( "github.com/apache/arrow/go/v13/arrow/memory" api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) var _ ColumnarBuffer[any] = (*columnarBufferArrowIPCStreamingDefault[any])(nil) @@ -19,7 +19,7 @@ type columnarBufferArrowIPCStreamingDefault[T utils.Acceptor] struct { arrowAllocator memory.Allocator builders []array.Builder schema *arrow.Schema - logger log.Logger + logger *zap.Logger } // AddRow saves a row obtained from the datasource into the buffer diff --git a/app/server/paging/columnar_buffer_factory.go b/app/server/paging/columnar_buffer_factory.go index 69f106cf..2b921656 100644 --- a/app/server/paging/columnar_buffer_factory.go +++ b/app/server/paging/columnar_buffer_factory.go @@ -7,13 +7,13 @@ import ( "github.com/apache/arrow/go/v13/arrow/memory" api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" + "go.uber.org/zap" ) type columnarBufferFactoryImpl[T utils.Acceptor] struct { arrowAllocator memory.Allocator - logger log.Logger + logger *zap.Logger format api_service_protos.TReadSplitsRequest_EFormat schema *arrow.Schema ydbTypes []*Ydb.Type @@ -47,7 +47,7 @@ func (cbf *columnarBufferFactoryImpl[T]) MakeBuffer() (ColumnarBuffer[T], error) } func NewColumnarBufferFactory[T utils.Acceptor]( - logger log.Logger, + logger *zap.Logger, arrowAllocator memory.Allocator, format api_service_protos.TReadSplitsRequest_EFormat, selectWhat *api_service_protos.TSelect_TWhat, diff --git a/app/server/paging/read_limiter.go b/app/server/paging/read_limiter.go index 79189634..1d454741 100644 --- a/app/server/paging/read_limiter.go +++ b/app/server/paging/read_limiter.go @@ -5,7 +5,7 @@ import ( "github.com/ydb-platform/fq-connector-go/app/config" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) // ReadLimiter helps to limitate amount of data returned by Connector server in every read request. @@ -40,7 +40,7 @@ type ReadLimiterFactory struct { cfg *config.TServerReadLimit } -func (rlf *ReadLimiterFactory) MakeReadLimiter(logger log.Logger) ReadLimiter { +func (rlf *ReadLimiterFactory) MakeReadLimiter(logger *zap.Logger) ReadLimiter { if rlf.cfg == nil { return readLimiterNoop{} } diff --git a/app/server/paging/sink.go b/app/server/paging/sink.go index 65f9717d..644c4d29 100644 --- a/app/server/paging/sink.go +++ b/app/server/paging/sink.go @@ -7,7 +7,7 @@ import ( api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) type sinkState int8 @@ -27,7 +27,7 @@ type sinkImpl[T utils.Acceptor] struct { bufferFactory ColumnarBufferFactory[T] // creates new buffer trafficTracker *TrafficTracker[T] // tracks the amount of data passed through the sink readLimiter ReadLimiter // helps to restrict the number of rows read in every request - logger log.Logger // annotated logger + logger *zap.Logger // annotated logger state sinkState // flag showing if it's ready to return data ctx context.Context // client context } @@ -146,7 +146,7 @@ func (s *sinkImpl[T]) unexpectedState(expected ...sinkState) error { func NewSink[T utils.Acceptor]( ctx context.Context, - logger log.Logger, + logger *zap.Logger, trafficTracker *TrafficTracker[T], columnarBufferFactory ColumnarBufferFactory[T], readLimiter ReadLimiter, diff --git a/app/server/service_connector.go b/app/server/service_connector.go index a224c025..bf2ebf45 100644 --- a/app/server/service_connector.go +++ b/app/server/service_connector.go @@ -13,8 +13,8 @@ import ( "github.com/ydb-platform/fq-connector-go/app/config" "github.com/ydb-platform/fq-connector-go/app/server/paging" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/fq-connector-go/library/go/core/metrics/solomon" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/credentials" "google.golang.org/grpc/reflection" @@ -26,7 +26,7 @@ type serviceConnector struct { cfg *config.TServerConfig grpcServer *grpc.Server listener net.Listener - logger log.Logger + logger *zap.Logger } func (s *serviceConnector) ListTables(_ *api_service_protos.TListTablesRequest, _ api_service.Connector_ListTablesServer) error { @@ -38,10 +38,10 @@ func (s *serviceConnector) DescribeTable( request *api_service_protos.TDescribeTableRequest, ) (*api_service_protos.TDescribeTableResponse, error) { logger := utils.AnnotateLogger(s.logger, "DescribeTable", request.DataSourceInstance) - logger.Info("request handling started", log.String("table", request.GetTable())) + logger.Info("request handling started", zap.String("table", request.GetTable())) if err := ValidateDescribeTableRequest(logger, request); err != nil { - logger.Error("request handling failed", log.Error(err)) + logger.Error("request handling failed", zap.Error(err)) return &api_service_protos.TDescribeTableResponse{ Error: utils.NewAPIErrorFromStdError(err), @@ -50,7 +50,7 @@ func (s *serviceConnector) DescribeTable( out, err := s.dataSourceCollection.DescribeTable(ctx, logger, request) if err != nil { - logger.Error("request handling failed", log.Error(err)) + logger.Error("request handling failed", zap.Error(err)) out = &api_service_protos.TDescribeTableResponse{Error: utils.NewAPIErrorFromStdError(err)} @@ -58,14 +58,14 @@ func (s *serviceConnector) DescribeTable( } out.Error = utils.NewSuccess() - logger.Info("request handling finished", log.String("response", out.String())) + logger.Info("request handling finished", zap.String("response", out.String())) return out, nil } func (s *serviceConnector) ListSplits(request *api_service_protos.TListSplitsRequest, stream api_service.Connector_ListSplitsServer) error { logger := utils.AnnotateLogger(s.logger, "ListSplits", nil) - logger.Info("request handling started", log.Int("total selects", len(request.Selects))) + logger.Info("request handling started", zap.Int("total selects", len(request.Selects))) if err := ValidateListSplitsRequest(logger, request); err != nil { return s.doListSplitsResponse(logger, stream, @@ -77,13 +77,13 @@ func (s *serviceConnector) ListSplits(request *api_service_protos.TListSplitsReq for _, slct := range request.Selects { if err := s.doListSplitsHandleSelect(stream, slct, &totalSplits); err != nil { - logger.Error("request handling failed", log.Error(err)) + logger.Error("request handling failed", zap.Error(err)) return err } } - logger.Info("request handling finished", log.Int("total_splits", totalSplits)) + logger.Info("request handling finished", zap.Int("total_splits", totalSplits)) return nil } @@ -95,8 +95,8 @@ func (s *serviceConnector) doListSplitsHandleSelect( ) error { logger := utils.AnnotateLogger(s.logger, "ListSplits", slct.DataSourceInstance) - args := []log.Field{ - log.Int("split_id", *totalSplits), + args := []zap.Field{ + zap.Int("split_id", *totalSplits), } args = append(args, utils.SelectToFields(slct)...) @@ -108,8 +108,8 @@ func (s *serviceConnector) doListSplitsHandleSelect( } for _, split := range resp.Splits { - args := []log.Field{ - log.Int("split_id", *totalSplits), + args := []zap.Field{ + zap.Int("split_id", *totalSplits), } args = append(args, utils.SelectToFields(split.Select)...) @@ -126,7 +126,7 @@ func (s *serviceConnector) doListSplitsHandleSelect( } func (s *serviceConnector) doListSplitsResponse( - logger log.Logger, + logger *zap.Logger, stream api_service.Connector_ListSplitsServer, response *api_service_protos.TListSplitsResponse, ) error { @@ -135,7 +135,7 @@ func (s *serviceConnector) doListSplitsResponse( } if err := stream.Send(response); err != nil { - logger.Error("send channel failed", log.Error(err)) + logger.Error("send channel failed", zap.Error(err)) return err } @@ -147,11 +147,11 @@ func (s *serviceConnector) ReadSplits( request *api_service_protos.TReadSplitsRequest, stream api_service.Connector_ReadSplitsServer) error { logger := utils.AnnotateLogger(s.logger, "ReadSplits", request.DataSourceInstance) - logger.Info("request handling started", log.Int("total_splits", len(request.Splits))) + logger.Info("request handling started", zap.Int("total_splits", len(request.Splits))) err := s.doReadSplits(logger, request, stream) if err != nil { - logger.Error("request handling failed", log.Error(err)) + logger.Error("request handling failed", zap.Error(err)) response := &api_service_protos.TReadSplitsResponse{Error: utils.NewAPIErrorFromStdError(err)} @@ -166,7 +166,7 @@ func (s *serviceConnector) ReadSplits( } func (s *serviceConnector) doReadSplits( - logger log.Logger, + logger *zap.Logger, request *api_service_protos.TReadSplitsRequest, stream api_service.Connector_ReadSplitsServer, ) error { @@ -175,7 +175,7 @@ func (s *serviceConnector) doReadSplits( } for i, split := range request.Splits { - splitLogger := log.With(logger, log.Int("split_id", i)) + splitLogger := logger.With(zap.Int("split_id", i)) err := s.dataSourceCollection.DoReadSplit( splitLogger, @@ -193,7 +193,7 @@ func (s *serviceConnector) doReadSplits( } func (s *serviceConnector) start() error { - s.logger.Debug("starting GRPC server", log.String("address", s.listener.Addr().String())) + s.logger.Debug("starting GRPC server", zap.String("address", s.listener.Addr().String())) if err := s.grpcServer.Serve(s.listener); err != nil { return fmt.Errorf("listener serve: %w", err) @@ -202,7 +202,7 @@ func (s *serviceConnector) start() error { return nil } -func makeGRPCOptions(logger log.Logger, cfg *config.TServerConfig, registry *solomon.Registry) ([]grpc.ServerOption, error) { +func makeGRPCOptions(logger *zap.Logger, cfg *config.TServerConfig, registry *solomon.Registry) ([]grpc.ServerOption, error) { var ( opts []grpc.ServerOption tlsConfig *config.TServerTLSConfig @@ -228,7 +228,7 @@ func makeGRPCOptions(logger log.Logger, cfg *config.TServerConfig, registry *sol logger.Info("server will use TLS connections") - logger.Debug("reading key pair", log.String("cert", tlsConfig.Cert), log.String("key", tlsConfig.Key)) + logger.Debug("reading key pair", zap.String("cert", tlsConfig.Cert), zap.String("key", tlsConfig.Key)) cert, err := tls.LoadX509KeyPair(tlsConfig.Cert, tlsConfig.Key) if err != nil { @@ -247,7 +247,7 @@ func (s *serviceConnector) stop() { } func newServiceConnector( - logger log.Logger, + logger *zap.Logger, cfg *config.TServerConfig, registry *solomon.Registry, ) (service, error) { diff --git a/app/server/service_metrics.go b/app/server/service_metrics.go index f672d618..ad9f1360 100644 --- a/app/server/service_metrics.go +++ b/app/server/service_metrics.go @@ -7,18 +7,18 @@ import ( "github.com/ydb-platform/fq-connector-go/app/config" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/fq-connector-go/library/go/core/metrics/solomon" + "go.uber.org/zap" ) type serviceMetrics struct { httpServer *http.Server - logger log.Logger + logger *zap.Logger registry *solomon.Registry } func (s *serviceMetrics) start() error { - s.logger.Debug("starting HTTP metrics server", log.String("address", s.httpServer.Addr)) + s.logger.Debug("starting HTTP metrics server", zap.String("address", s.httpServer.Addr)) if err := s.httpServer.ListenAndServe(); err != nil { return fmt.Errorf("http metrics server listen and serve: %w", err) @@ -33,13 +33,13 @@ func (s *serviceMetrics) stop() { err := s.httpServer.Shutdown(ctx) if err != nil { - s.logger.Error("shutdown http metrics server", log.Error(err)) + s.logger.Error("shutdown http metrics server", zap.Error(err)) } } -func newServiceMetrics(logger log.Logger, cfg *config.TMetricsServerConfig, registry *solomon.Registry) service { +func newServiceMetrics(logger *zap.Logger, cfg *config.TMetricsServerConfig, registry *solomon.Registry) service { mux := http.NewServeMux() - mux.Handle("/metrics", NewHTTPPullerHandler(registry, WithSpack())) + mux.Handle("/metrics", NewHTTPPullerHandler(logger, registry, WithSpack())) httpServer := &http.Server{ Addr: utils.EndpointToString(cfg.Endpoint), diff --git a/app/server/service_pprof.go b/app/server/service_pprof.go index 7970cd5e..15946f23 100644 --- a/app/server/service_pprof.go +++ b/app/server/service_pprof.go @@ -9,16 +9,16 @@ import ( "github.com/ydb-platform/fq-connector-go/app/config" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) type servicePprof struct { httpServer *http.Server - logger log.Logger + logger *zap.Logger } func (s *servicePprof) start() error { - s.logger.Debug("starting HTTP server", log.String("address", s.httpServer.Addr)) + s.logger.Debug("starting HTTP server", zap.String("address", s.httpServer.Addr)) if err := s.httpServer.ListenAndServe(); err != nil { return fmt.Errorf("http server listen and server: %w", err) @@ -35,11 +35,11 @@ func (s *servicePprof) stop() { err := s.httpServer.Shutdown(ctx) if err != nil { - s.logger.Error("shutdown http server", log.Error(err)) + s.logger.Error("shutdown http server", zap.Error(err)) } } -func newServicePprof(logger log.Logger, cfg *config.TPprofServerConfig) service { +func newServicePprof(logger *zap.Logger, cfg *config.TPprofServerConfig) service { mux := http.NewServeMux() mux.HandleFunc("/debug/pprof/", pprof.Index) mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline) diff --git a/app/server/streaming/streamer.go b/app/server/streaming/streamer.go index dd7ea49e..a6bd3b0f 100644 --- a/app/server/streaming/streamer.go +++ b/app/server/streaming/streamer.go @@ -10,7 +10,7 @@ import ( "github.com/ydb-platform/fq-connector-go/app/server/datasource" "github.com/ydb-platform/fq-connector-go/app/server/paging" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) type Streamer[T utils.Acceptor] struct { @@ -19,7 +19,7 @@ type Streamer[T utils.Acceptor] struct { dataSource datasource.DataSource[T] split *api_service_protos.TSplit sink paging.Sink[T] - logger log.Logger + logger *zap.Logger ctx context.Context // clone of a stream context cancel context.CancelFunc } @@ -95,7 +95,7 @@ func (s *Streamer[T]) Run() error { } func NewStreamer[T utils.Acceptor]( - logger log.Logger, + logger *zap.Logger, stream api_service.Connector_ReadSplitsServer, request *api_service_protos.TReadSplitsRequest, split *api_service_protos.TSplit, diff --git a/app/server/streaming/streamer_test.go b/app/server/streaming/streamer_test.go index ffeac8d4..03cc3261 100644 --- a/app/server/streaming/streamer_test.go +++ b/app/server/streaming/streamer_test.go @@ -21,8 +21,8 @@ import ( rdbms_utils "github.com/ydb-platform/fq-connector-go/app/server/datasource/rdbms/utils" "github.com/ydb-platform/fq-connector-go/app/server/paging" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" + "go.uber.org/zap" ) var _ api_service.Connector_ReadSplitsServer = (*streamMock)(nil) @@ -30,7 +30,7 @@ var _ api_service.Connector_ReadSplitsServer = (*streamMock)(nil) type streamMock struct { mock.Mock api_service.Connector_ReadSplitsServer - logger log.Logger + logger *zap.Logger } func (m *streamMock) Context() context.Context { diff --git a/app/server/utils/errors.go b/app/server/utils/errors.go index a066cd8c..a9980624 100644 --- a/app/server/utils/errors.go +++ b/app/server/utils/errors.go @@ -5,8 +5,8 @@ import ( "fmt" api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" "github.com/ydb-platform/ydb-go-genproto/protos/Ydb" + "go.uber.org/zap" ) var ( @@ -91,10 +91,10 @@ func NewAPIErrorFromStdError(err error) *api_service_protos.TError { } } -func APIErrorToLogFields(apiErr *api_service_protos.TError) []log.Field { - return []log.Field{ - log.String("message", apiErr.Message), - log.String("status", apiErr.Status.String()), +func APIErrorToLogFields(apiErr *api_service_protos.TError) []zap.Field { + return []zap.Field{ + zap.String("message", apiErr.Message), + zap.String("status", apiErr.Status.String()), } } diff --git a/app/server/utils/logger.go b/app/server/utils/logger.go index 22233fb4..8e5f26a8 100644 --- a/app/server/utils/logger.go +++ b/app/server/utils/logger.go @@ -8,25 +8,24 @@ import ( api_common "github.com/ydb-platform/fq-connector-go/api/common" api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" "github.com/ydb-platform/fq-connector-go/app/config" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "github.com/ydb-platform/fq-connector-go/library/go/core/log/zap" + "go.uber.org/zap" "go.uber.org/zap/zapcore" "go.uber.org/zap/zaptest" ) // TODO: it's better to do this in GRPC middleware -func AnnotateLogger(logger log.Logger, method string, dsi *api_common.TDataSourceInstance) log.Logger { - logger = log.With(logger, log.String("method", method)) +func AnnotateLogger(l *zap.Logger, method string, dsi *api_common.TDataSourceInstance) *zap.Logger { + logger := l.With(zap.String("method", method)) if dsi != nil { - logger = log.With(logger, - log.String("data_source_kind", api_common.EDataSourceKind_name[int32(dsi.Kind)]), - log.String("host", dsi.Endpoint.Host), - log.UInt32("port", dsi.Endpoint.Port), - log.String("database", dsi.Database), - log.Bool("use_tls", dsi.UseTls), - log.String("protocol", dsi.Protocol.String()), + logger = logger.With( + zap.String("data_source_kind", api_common.EDataSourceKind_name[int32(dsi.Kind)]), + zap.String("host", dsi.Endpoint.Host), + zap.Uint32("port", dsi.Endpoint.Port), + zap.String("database", dsi.Database), + zap.Bool("use_tls", dsi.UseTls), + zap.String("protocol", dsi.Protocol.String()), // TODO: can we print just a login without a password? ) } @@ -34,20 +33,21 @@ func AnnotateLogger(logger log.Logger, method string, dsi *api_common.TDataSourc return logger } -func LogCloserError(logger log.Logger, closer io.Closer, msg string) { +func LogCloserError(logger *zap.Logger, closer io.Closer, msg string) { if err := closer.Close(); err != nil { - logger.Error(msg, log.Error(err)) + logger.Error(msg, zap.Error(err)) } } -func NewLoggerFromConfig(cfg *config.TLoggerConfig) (log.Logger, error) { +func NewLoggerFromConfig(cfg *config.TLoggerConfig) (*zap.Logger, error) { if cfg == nil { return NewDefaultLogger() } - loggerCfg := zap.NewDeployConfig() + loggerCfg := zap.NewProductionConfig() loggerCfg.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder loggerCfg.Encoding = "console" + loggerCfg.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder loggerCfg.Level.SetLevel(convertToZapLogLevel(cfg.GetLogLevel())) zapLogger, err := loggerCfg.Build() @@ -55,38 +55,38 @@ func NewLoggerFromConfig(cfg *config.TLoggerConfig) (log.Logger, error) { return nil, fmt.Errorf("new logger: %w", err) } - return &zap.Logger{L: zapLogger}, nil + return zapLogger, nil } -func NewDefaultLogger() (log.Logger, error) { - return NewLoggerFromConfig(&config.TLoggerConfig{LogLevel: config.ELogLevel_TRACE}) +func NewDefaultLogger() (*zap.Logger, error) { + return zap.Must(zap.NewDevelopment()), nil } -func NewTestLogger(t *testing.T) log.Logger { return &zap.Logger{L: zaptest.NewLogger(t)} } +func NewTestLogger(t *testing.T) *zap.Logger { return zaptest.NewLogger(t) } -func DumpReadSplitsResponse(logger log.Logger, resp *api_service_protos.TReadSplitsResponse) { +func DumpReadSplitsResponse(logger *zap.Logger, resp *api_service_protos.TReadSplitsResponse) { switch t := resp.GetPayload().(type) { case *api_service_protos.TReadSplitsResponse_ArrowIpcStreaming: if dump := resp.GetArrowIpcStreaming(); dump != nil { - logger.Debug("response", log.Int("arrow_blob_length", len(dump))) + logger.Debug("response", zap.Int("arrow_blob_length", len(dump))) } case *api_service_protos.TReadSplitsResponse_ColumnSet: for i := range t.ColumnSet.Data { data := t.ColumnSet.Data[i] meta := t.ColumnSet.Meta[i] - logger.Debug("response", log.Int("column_id", i), log.String("meta", meta.String()), log.String("data", data.String())) + logger.Debug("response", zap.Int("column_id", i), zap.String("meta", meta.String()), zap.String("data", data.String())) } default: panic(fmt.Sprintf("unexpected message type %v", t)) } } -func SelectToFields(slct *api_service_protos.TSelect) []log.Field { - result := []log.Field{ - log.Any("from", slct.From), - log.Any("what", slct.What), - log.Any("where", slct.Where), +func SelectToFields(slct *api_service_protos.TSelect) []zap.Field { + result := []zap.Field{ + zap.Any("from", slct.From), + zap.Any("what", slct.What), + zap.Any("where", slct.Where), } return result @@ -102,12 +102,12 @@ func NewQueryLoggerFactory(cfg *config.TLoggerConfig) QueryLoggerFactory { return QueryLoggerFactory{enableQueryLogging: enabled} } -func (f *QueryLoggerFactory) Make(logger log.Logger) QueryLogger { +func (f *QueryLoggerFactory) Make(logger *zap.Logger) QueryLogger { return QueryLogger{Logger: logger, enabled: f.enableQueryLogging} } type QueryLogger struct { - log.Logger + *zap.Logger enabled bool } @@ -116,9 +116,9 @@ func (ql *QueryLogger) Dump(query string, args ...any) { return } - logFields := []log.Field{log.String("query", query)} + logFields := []zap.Field{zap.String("query", query)} if len(args) > 0 { - logFields = append(logFields, log.Any("args", args)) + logFields = append(logFields, zap.Any("args", args)) } ql.Debug("execute SQL query", logFields...) diff --git a/app/server/validate.go b/app/server/validate.go index e9fdac6a..45a994da 100644 --- a/app/server/validate.go +++ b/app/server/validate.go @@ -6,10 +6,10 @@ import ( api_common "github.com/ydb-platform/fq-connector-go/api/common" api_service_protos "github.com/ydb-platform/fq-connector-go/api/service/protos" "github.com/ydb-platform/fq-connector-go/app/server/utils" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" + "go.uber.org/zap" ) -func ValidateDescribeTableRequest(logger log.Logger, request *api_service_protos.TDescribeTableRequest) error { +func ValidateDescribeTableRequest(logger *zap.Logger, request *api_service_protos.TDescribeTableRequest) error { if err := validateDataSourceInstance(logger, request.GetDataSourceInstance()); err != nil { return fmt.Errorf("validate data source instance: %w", err) } @@ -21,7 +21,7 @@ func ValidateDescribeTableRequest(logger log.Logger, request *api_service_protos return nil } -func ValidateListSplitsRequest(logger log.Logger, request *api_service_protos.TListSplitsRequest) error { +func ValidateListSplitsRequest(logger *zap.Logger, request *api_service_protos.TListSplitsRequest) error { if len(request.Selects) == 0 { return fmt.Errorf("empty select list: %w", utils.ErrInvalidRequest) } @@ -35,7 +35,7 @@ func ValidateListSplitsRequest(logger log.Logger, request *api_service_protos.TL return nil } -func ValidateReadSplitsRequest(logger log.Logger, request *api_service_protos.TReadSplitsRequest) error { +func ValidateReadSplitsRequest(logger *zap.Logger, request *api_service_protos.TReadSplitsRequest) error { if err := validateDataSourceInstance(logger, request.GetDataSourceInstance()); err != nil { return fmt.Errorf("validate data source instance: %w", err) } @@ -47,7 +47,7 @@ func ValidateReadSplitsRequest(logger log.Logger, request *api_service_protos.TR return nil } -func validateSelect(logger log.Logger, slct *api_service_protos.TSelect) error { +func validateSelect(logger *zap.Logger, slct *api_service_protos.TSelect) error { if slct == nil { return fmt.Errorf("select is empty: %w", utils.ErrInvalidRequest) } @@ -59,7 +59,7 @@ func validateSelect(logger log.Logger, slct *api_service_protos.TSelect) error { return nil } -func validateDataSourceInstance(logger log.Logger, dsi *api_common.TDataSourceInstance) error { +func validateDataSourceInstance(logger *zap.Logger, dsi *api_common.TDataSourceInstance) error { if dsi.GetKind() == api_common.EDataSourceKind_DATA_SOURCE_KIND_UNSPECIFIED { return fmt.Errorf("empty kind: %w", utils.ErrInvalidRequest) } diff --git a/library/go/core/log/compat/golog/log.go b/library/go/core/log/compat/golog/log.go deleted file mode 100644 index 3edd7e96..00000000 --- a/library/go/core/log/compat/golog/log.go +++ /dev/null @@ -1,21 +0,0 @@ -package golog - -import ( - canal_log "github.com/siddontang/go-log/log" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" -) - -func SetLevel(level log.Level) { - switch level { - case log.DebugLevel: - canal_log.SetLevel(canal_log.LevelDebug) - case log.ErrorLevel: - canal_log.SetLevel(canal_log.LevelError) - case log.FatalLevel: - canal_log.SetLevel(canal_log.LevelFatal) - case log.InfoLevel: - canal_log.SetLevel(canal_log.LevelInfo) - case log.TraceLevel: - canal_log.SetLevel(canal_log.LevelTrace) - } -} diff --git a/library/go/core/log/compat/logrus/log.go b/library/go/core/log/compat/logrus/log.go deleted file mode 100644 index 0ebd6a15..00000000 --- a/library/go/core/log/compat/logrus/log.go +++ /dev/null @@ -1,202 +0,0 @@ -package logrus - -import ( - "io" - "runtime" - "strings" - "sync" - - "github.com/sirupsen/logrus" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" -) - -/* Call frame calculations are copied from logrus package */ -var ( - - // qualified package name, cached at first use - logrusPackage string - - // Positions in the call stack when tracing to report the calling method - minimumCallerDepth int - - // Used for caller information initialisation - callerInitOnce sync.Once -) - -const ( - maximumCallerDepth int = 25 - knownLogrusFrames int = 4 -) - -func init() { - // start at the bottom of the stack before the package-name cache is primed - minimumCallerDepth = 1 -} - -// getPackageName reduces a fully qualified function name to the package name -// There really ought to be to be a better way... -func getPackageName(f string) string { - for { - lastPeriod := strings.LastIndex(f, ".") - lastSlash := strings.LastIndex(f, "/") - if lastPeriod > lastSlash { - f = f[:lastPeriod] - } else { - break - } - } - - return f -} - -func getCallerDepth() int { - // cache this package's fully-qualified name - callerInitOnce.Do(func() { - pcs := make([]uintptr, maximumCallerDepth) - _ = runtime.Callers(0, pcs) - - // dynamic get the package name and the minimum caller depth - logrusIsNext := false - for i := 0; i < maximumCallerDepth; i++ { - funcName := runtime.FuncForPC(pcs[i]).Name() - if logrusIsNext { - logrusPackage = getPackageName(funcName) - break - } - if strings.Contains(funcName, "LogrusAdapter") { - logrusIsNext = true - continue - } - } - - minimumCallerDepth = knownLogrusFrames - }) - - // Restrict the lookback frames to avoid runaway lookups - pcs := make([]uintptr, maximumCallerDepth) - depth := runtime.Callers(minimumCallerDepth, pcs) - frames := runtime.CallersFrames(pcs[:depth]) - callerDepth := minimumCallerDepth - - for f, again := frames.Next(); again; f, again = frames.Next() { - pkg := getPackageName(f.Function) - - // If the caller isn't part of this package, we're done - if pkg != logrusPackage { - return callerDepth - 2 - } - callerDepth++ - } - - // if we got here, we failed to find the caller's context - return 0 -} - -func convertLevel(level log.Level) logrus.Level { - switch level { - case log.TraceLevel: - return logrus.TraceLevel - case log.DebugLevel: - return logrus.DebugLevel - case log.InfoLevel: - return logrus.InfoLevel - case log.WarnLevel: - return logrus.WarnLevel - case log.ErrorLevel: - return logrus.ErrorLevel - case log.FatalLevel: - return logrus.FatalLevel - } - - return logrus.PanicLevel -} - -func SetLevel(level log.Level) { - logrus.SetLevel(convertLevel(level)) -} - -type LogrusAdapter struct { - logger log.Logger - adaptCallstack bool - convertPrefix bool -} - -func (a *LogrusAdapter) Format(entry *logrus.Entry) ([]byte, error) { - var name *string - fields := make([]log.Field, 0, len(entry.Data)) - for key, val := range entry.Data { - skip := false - if a.convertPrefix && key == "prefix" { - if w, ok := val.(string); ok { - name = &w - skip = true - } - } - if !skip { - fields = append(fields, log.Any(key, val)) - } - } - - var logger log.Logger - if a.adaptCallstack { - logger = log.AddCallerSkip(a.logger, getCallerDepth()) - } else { - logger = a.logger - } - - if a.convertPrefix && name != nil { - logger = logger.WithName(*name) - } - - switch entry.Level { - case logrus.TraceLevel: - logger.Trace(entry.Message, fields...) - case logrus.DebugLevel: - logger.Debug(entry.Message, fields...) - case logrus.InfoLevel: - logger.Info(entry.Message, fields...) - case logrus.WarnLevel: - logger.Warn(entry.Message, fields...) - case logrus.ErrorLevel: - logger.Error(entry.Message, fields...) - case logrus.FatalLevel: - logger.Fatal(entry.Message, fields...) - case logrus.PanicLevel: - logger.Fatal(entry.Message, fields...) - } - - return nil, nil -} - -type Option func(*LogrusAdapter) - -func DontAdaptCallstack() Option { - return func(adapter *LogrusAdapter) { - adapter.adaptCallstack = false - } -} - -func ConvertPrefix() Option { - return func(adapter *LogrusAdapter) { - adapter.convertPrefix = true - } -} - -// AdaptLogrus replaces logr formatter by wrapped logger -func AdaptLogrus(logr *logrus.Logger, logger log.Logger, level log.Level, opts ...Option) { - logr.SetLevel(convertLevel(level)) - - adapter := &LogrusAdapter{logger, true, false} - - for _, opt := range opts { - opt(adapter) - } - - logr.SetFormatter(adapter) - logr.SetOutput(io.Discard) -} - -// AdaptStandardLogger replaces logrus.StandardLogger() formatter by wrapped logger -func AdaptStandardLogger(logger log.Logger, level log.Level, opts ...Option) { - AdaptLogrus(logrus.StandardLogger(), logger, level, opts...) -} diff --git a/library/go/core/log/compat/pion/log.go b/library/go/core/log/compat/pion/log.go deleted file mode 100644 index e0905615..00000000 --- a/library/go/core/log/compat/pion/log.go +++ /dev/null @@ -1,76 +0,0 @@ -package pion - -import ( - "github.com/pion/logging" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" -) - -type LoggerFactory struct { - StandardLogger log.Logger -} - -func (l LoggerFactory) NewLogger(scope string) logging.LeveledLogger { - return LoggerAdapter{ - standardLogger: l.StandardLogger, - scope: scope, - } -} - -type LoggerAdapter struct { - standardLogger log.Logger - scope string -} - -func (a LoggerAdapter) Trace(msg string) { - log.AddCallerSkip(a.standardLogger, 1) - a.standardLogger.Trace(a.addScope(msg)) -} - -func (a LoggerAdapter) Tracef(format string, args ...interface{}) { - log.AddCallerSkip(a.standardLogger, 1) - a.standardLogger.Tracef(a.addScope(format), args...) -} - -func (a LoggerAdapter) Debug(msg string) { - log.AddCallerSkip(a.standardLogger, 1) - a.standardLogger.Debug(a.addScope(msg)) -} - -func (a LoggerAdapter) Debugf(format string, args ...interface{}) { - log.AddCallerSkip(a.standardLogger, 1) - a.standardLogger.Debugf(a.addScope(format), args...) -} - -func (a LoggerAdapter) Info(msg string) { - log.AddCallerSkip(a.standardLogger, 1) - a.standardLogger.Info(a.addScope(msg)) -} - -func (a LoggerAdapter) Infof(format string, args ...interface{}) { - log.AddCallerSkip(a.standardLogger, 1) - a.standardLogger.Infof(a.addScope(format), args...) -} - -func (a LoggerAdapter) Warn(msg string) { - log.AddCallerSkip(a.standardLogger, 1) - a.standardLogger.Warn(a.addScope(msg)) -} - -func (a LoggerAdapter) Warnf(format string, args ...interface{}) { - log.AddCallerSkip(a.standardLogger, 1) - a.standardLogger.Warnf(a.addScope(format), args...) -} - -func (a LoggerAdapter) Error(msg string) { - log.AddCallerSkip(a.standardLogger, 1) - a.standardLogger.Error(a.addScope(msg)) -} - -func (a LoggerAdapter) Errorf(format string, args ...interface{}) { - log.AddCallerSkip(a.standardLogger, 1) - a.standardLogger.Errorf(a.addScope(format), args...) -} - -func (a LoggerAdapter) addScope(s string) string { - return a.scope + ": " + s -} diff --git a/library/go/core/log/compat/stdlog/stdlog.go b/library/go/core/log/compat/stdlog/stdlog.go deleted file mode 100644 index a48e15b4..00000000 --- a/library/go/core/log/compat/stdlog/stdlog.go +++ /dev/null @@ -1,54 +0,0 @@ -package stdlog - -import ( - "bytes" - "fmt" - stdlog "log" - - "github.com/ydb-platform/fq-connector-go/library/go/core/log" -) - -func levelToFunc(logger log.Logger, lvl log.Level) (func(msg string, fields ...log.Field), error) { - switch lvl { - case log.DebugLevel: - return logger.Debug, nil - case log.TraceLevel: - return logger.Trace, nil - case log.InfoLevel: - return logger.Info, nil - case log.WarnLevel: - return logger.Warn, nil - case log.ErrorLevel: - return logger.Error, nil - case log.FatalLevel: - return logger.Fatal, nil - } - - return nil, fmt.Errorf("unknown log level: %v", lvl) -} - -type loggerWriter struct { - logFunc func(msg string, fields ...log.Field) -} - -func (w *loggerWriter) Write(p []byte) (int, error) { - p = bytes.TrimSpace(p) - w.logFunc(string(p)) - return len(p), nil -} - -// New creates stdlib log.Logger that writes to provided logger on Error level -func New(logger log.Logger) *stdlog.Logger { - l := log.AddCallerSkip(logger, 3) - return stdlog.New(&loggerWriter{logFunc: l.Error}, "", 0) -} - -// NewAt creates stdlib log.Logger that writes to provided logger on specified level -func NewAt(logger log.Logger, lvl log.Level) (*stdlog.Logger, error) { - l := log.AddCallerSkip(logger, 3) - logFunc, err := levelToFunc(l, lvl) - if err != nil { - return nil, err - } - return stdlog.New(&loggerWriter{logFunc: logFunc}, "", 0), nil -} diff --git a/library/go/core/log/ctxlog/ctxlog.go b/library/go/core/log/ctxlog/ctxlog.go deleted file mode 100644 index e90ebc72..00000000 --- a/library/go/core/log/ctxlog/ctxlog.go +++ /dev/null @@ -1,124 +0,0 @@ -package ctxlog - -import ( - "context" - "fmt" - - "github.com/ydb-platform/fq-connector-go/library/go/core/log" -) - -type ctxKey struct{} - -// ContextFields returns log.Fields bound with ctx. -// If no fields are bound, it returns nil. -func ContextFields(ctx context.Context) []log.Field { - fs, _ := ctx.Value(ctxKey{}).([]log.Field) - return fs -} - -// WithFields returns a new context that is bound with given fields and based -// on parent ctx. -func WithFields(ctx context.Context, fields ...log.Field) context.Context { - if len(fields) == 0 { - return ctx - } - - return context.WithValue(ctx, ctxKey{}, mergeFields(ContextFields(ctx), fields)) -} - -// Trace logs at Trace log level using fields both from arguments and ones that -// are bound to ctx. -func Trace(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { - log.AddCallerSkip(l, 1).Trace(msg, mergeFields(ContextFields(ctx), fields)...) -} - -// Debug logs at Debug log level using fields both from arguments and ones that -// are bound to ctx. -func Debug(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { - log.AddCallerSkip(l, 1).Debug(msg, mergeFields(ContextFields(ctx), fields)...) -} - -// Info logs at Info log level using fields both from arguments and ones that -// are bound to ctx. -func Info(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { - log.AddCallerSkip(l, 1).Info(msg, mergeFields(ContextFields(ctx), fields)...) -} - -// Warn logs at Warn log level using fields both from arguments and ones that -// are bound to ctx. -func Warn(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { - log.AddCallerSkip(l, 1).Warn(msg, mergeFields(ContextFields(ctx), fields)...) -} - -// Error logs at Error log level using fields both from arguments and ones that -// are bound to ctx. -func Error(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { - log.AddCallerSkip(l, 1).Error(msg, mergeFields(ContextFields(ctx), fields)...) -} - -// Fatal logs at Fatal log level using fields both from arguments and ones that -// are bound to ctx. -func Fatal(ctx context.Context, l log.Logger, msg string, fields ...log.Field) { - log.AddCallerSkip(l, 1).Fatal(msg, mergeFields(ContextFields(ctx), fields)...) -} - -// Tracef logs at Trace log level using fields that are bound to ctx. -// The message is formatted using provided arguments. -func Tracef(ctx context.Context, l log.Logger, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - log.AddCallerSkip(l, 1).Trace(msg, ContextFields(ctx)...) -} - -// Debugf logs at Debug log level using fields that are bound to ctx. -// The message is formatted using provided arguments. -func Debugf(ctx context.Context, l log.Logger, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - log.AddCallerSkip(l, 1).Debug(msg, ContextFields(ctx)...) -} - -// Infof logs at Info log level using fields that are bound to ctx. -// The message is formatted using provided arguments. -func Infof(ctx context.Context, l log.Logger, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - log.AddCallerSkip(l, 1).Info(msg, ContextFields(ctx)...) -} - -// Warnf logs at Warn log level using fields that are bound to ctx. -// The message is formatted using provided arguments. -func Warnf(ctx context.Context, l log.Logger, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - log.AddCallerSkip(l, 1).Warn(msg, ContextFields(ctx)...) -} - -// Errorf logs at Error log level using fields that are bound to ctx. -// The message is formatted using provided arguments. -func Errorf(ctx context.Context, l log.Logger, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - log.AddCallerSkip(l, 1).Error(msg, ContextFields(ctx)...) -} - -// Fatalf logs at Fatal log level using fields that are bound to ctx. -// The message is formatted using provided arguments. -func Fatalf(ctx context.Context, l log.Logger, format string, args ...interface{}) { - msg := fmt.Sprintf(format, args...) - log.AddCallerSkip(l, 1).Fatal(msg, ContextFields(ctx)...) -} - -func mergeFields(a, b []log.Field) []log.Field { - if len(a) == 0 { - return b - } - if len(b) == 0 { - return a - } - - // NOTE: just append() here is unsafe. If a caller passed slice of fields - // followed by ... with capacity greater than length, then simultaneous - // logging will lead to a data race condition. - // - // See https://golang.org/ref/spec#Passing_arguments_to_..._parameters - c := make([]log.Field, len(a)+len(b)) - n := copy(c, a) - copy(c[n:], b) - return c -} diff --git a/library/go/core/log/ctxlog/ctxlog_test.go b/library/go/core/log/ctxlog/ctxlog_test.go deleted file mode 100644 index f47a512f..00000000 --- a/library/go/core/log/ctxlog/ctxlog_test.go +++ /dev/null @@ -1,66 +0,0 @@ -package ctxlog - -import ( - "context" - "reflect" - "testing" - - "github.com/ydb-platform/fq-connector-go/library/go/core/log" -) - -func TestContextFields(t *testing.T) { - for _, test := range []struct { - ctx context.Context - exp []log.Field - }{ - { - ctx: context.Background(), - exp: nil, - }, - { - ctx: contextWithFields( - log.String("foo", "bar"), - log.String("bar", "baz"), - ), - exp: []log.Field{ - log.String("foo", "bar"), - log.String("bar", "baz"), - }, - }, - } { - t.Run("", func(t *testing.T) { - act := ContextFields(test.ctx) - if exp := test.exp; !reflect.DeepEqual(act, exp) { - t.Fatalf( - "ContextFields() = %v; want %v", - act, exp, - ) - } - }) - } -} - -// TestWithFields tests the case when race condition may occur on adding fields -// to a bound field slice capable enough to store additional ones. -func TestWithFields(t *testing.T) { - fs := make([]log.Field, 2, 4) - fs[0] = log.String("a", "a") - fs[1] = log.String("b", "b") - - // Bind to ctx1 field slice with cap(fs) = 2. - ctx1 := WithFields(context.Background(), fs...) - - // Bind additional two fields to ctx2 that are able to fit the parent's - // ctx1 bound fields. - _ = WithFields(ctx1, log.String("c", "c"), log.String("d", "d")) - - var act, exp [2]log.Field // Expect to zero-values of Field. - copy(act[:], fs[2:4]) // Check the tail of initial slice. - if act != exp { - t.Fatalf("fields tail is non-empty: %v", act) - } -} - -func contextWithFields(fs ...log.Field) context.Context { - return context.WithValue(context.Background(), ctxKey{}, fs) -} diff --git a/library/go/core/log/fields.go b/library/go/core/log/fields.go deleted file mode 100644 index ab19d2c2..00000000 --- a/library/go/core/log/fields.go +++ /dev/null @@ -1,458 +0,0 @@ -package log - -import ( - "context" - "fmt" - "time" -) - -const ( - // DefaultErrorFieldName is the default field name used for errors - DefaultErrorFieldName = "error" -) - -// FieldType is a type of data Field can represent -type FieldType int - -const ( - // FieldTypeNil is for a pure nil - FieldTypeNil FieldType = iota - // FieldTypeString is for a string - FieldTypeString - // FieldTypeBinary is for a binary array - FieldTypeBinary - // FieldTypeBoolean is for boolean - FieldTypeBoolean - // FieldTypeSigned is for signed integers - FieldTypeSigned - // FieldTypeUnsigned is for unsigned integers - FieldTypeUnsigned - // FieldTypeFloat is for float - FieldTypeFloat - // FieldTypeTime is for time.Time - FieldTypeTime - // FieldTypeDuration is for time.Duration - FieldTypeDuration - // FieldTypeError is for an error - FieldTypeError - // FieldTypeArray is for an array of any type - FieldTypeArray - // FieldTypeAny is for any type - FieldTypeAny - // FieldTypeReflect is for unknown types - FieldTypeReflect - // FieldTypeByteString is for a bytes that can be represented as UTF-8 string - FieldTypeByteString - // FieldTypeContext wraps context for lazy context fields evaluation if possible - FieldTypeContext -) - -// Field stores one structured logging field -type Field struct { - key string - ftype FieldType - string string - signed int64 - unsigned uint64 - float float64 - iface interface{} -} - -// Key returns field key -func (f Field) Key() string { - return f.key -} - -// Type returns field type -func (f Field) Type() FieldType { - return f.ftype -} - -// String returns field string -func (f Field) String() string { - return f.string -} - -// Binary constructs field of []byte -func (f Field) Binary() []byte { - if f.iface == nil { - return nil - } - return f.iface.([]byte) -} - -// Bool returns field bool -func (f Field) Bool() bool { - return f.Signed() != 0 -} - -// Signed returns field int64 -func (f Field) Signed() int64 { - return f.signed -} - -// Unsigned returns field uint64 -func (f Field) Unsigned() uint64 { - return f.unsigned -} - -// Float returns field float64 -func (f Field) Float() float64 { - return f.float -} - -// Time returns field time.Time -func (f Field) Time() time.Time { - return time.Unix(0, f.signed) -} - -// Duration returns field time.Duration -func (f Field) Duration() time.Duration { - return time.Nanosecond * time.Duration(f.signed) -} - -// Error constructs field of error type -func (f Field) Error() error { - if f.iface == nil { - return nil - } - return f.iface.(error) -} - -// Interface returns field interface -func (f Field) Interface() interface{} { - return f.iface -} - -// Any returns contained data as interface{} -// nolint: gocyclo -func (f Field) Any() interface{} { - switch f.Type() { - case FieldTypeNil: - return nil - case FieldTypeString: - return f.String() - case FieldTypeBinary: - return f.Interface() - case FieldTypeBoolean: - return f.Bool() - case FieldTypeSigned: - return f.Signed() - case FieldTypeUnsigned: - return f.Unsigned() - case FieldTypeFloat: - return f.Float() - case FieldTypeTime: - return f.Time() - case FieldTypeDuration: - return f.Duration() - case FieldTypeError: - return f.Error() - case FieldTypeArray: - return f.Interface() - case FieldTypeAny: - return f.Interface() - case FieldTypeReflect: - return f.Interface() - case FieldTypeByteString: - return f.Interface() - case FieldTypeContext: - return f.Interface() - default: - // For when new field type is not added to this func - panic(fmt.Sprintf("unknown field type: %d", f.Type())) - } -} - -// Nil constructs field of nil type -func Nil(key string) Field { - return Field{key: key, ftype: FieldTypeNil} -} - -// String constructs field of string type -func String(key, value string) Field { - return Field{key: key, ftype: FieldTypeString, string: value} -} - -// Sprintf constructs field of string type with formatting -func Sprintf(key, format string, args ...interface{}) Field { - return Field{key: key, ftype: FieldTypeString, string: fmt.Sprintf(format, args...)} -} - -// Strings constructs Field from []string -func Strings(key string, value []string) Field { - return Array(key, value) -} - -// Binary constructs field of []byte type -func Binary(key string, value []byte) Field { - return Field{key: key, ftype: FieldTypeBinary, iface: value} -} - -// Bool constructs field of bool type -func Bool(key string, value bool) Field { - field := Field{key: key, ftype: FieldTypeBoolean} - if value { - field.signed = 1 - } else { - field.signed = 0 - } - - return field -} - -// Bools constructs Field from []bool -func Bools(key string, value []bool) Field { - return Array(key, value) -} - -// Int constructs Field from int -func Int(key string, value int) Field { - return Int64(key, int64(value)) -} - -// Ints constructs Field from []int -func Ints(key string, value []int) Field { - return Array(key, value) -} - -// Int8 constructs Field from int8 -func Int8(key string, value int8) Field { - return Int64(key, int64(value)) -} - -// Int8s constructs Field from []int8 -func Int8s(key string, value []int8) Field { - return Array(key, value) -} - -// Int16 constructs Field from int16 -func Int16(key string, value int16) Field { - return Int64(key, int64(value)) -} - -// Int16s constructs Field from []int16 -func Int16s(key string, value []int16) Field { - return Array(key, value) -} - -// Int32 constructs Field from int32 -func Int32(key string, value int32) Field { - return Int64(key, int64(value)) -} - -// Int32s constructs Field from []int32 -func Int32s(key string, value []int32) Field { - return Array(key, value) -} - -// Int64 constructs Field from int64 -func Int64(key string, value int64) Field { - return Field{key: key, ftype: FieldTypeSigned, signed: value} -} - -// Int64s constructs Field from []int64 -func Int64s(key string, value []int64) Field { - return Array(key, value) -} - -// UInt constructs Field from uint -func UInt(key string, value uint) Field { - return UInt64(key, uint64(value)) -} - -// UInts constructs Field from []uint -func UInts(key string, value []uint) Field { - return Array(key, value) -} - -// UInt8 constructs Field from uint8 -func UInt8(key string, value uint8) Field { - return UInt64(key, uint64(value)) -} - -// UInt8s constructs Field from []uint8 -func UInt8s(key string, value []uint8) Field { - return Array(key, value) -} - -// UInt16 constructs Field from uint16 -func UInt16(key string, value uint16) Field { - return UInt64(key, uint64(value)) -} - -// UInt16s constructs Field from []uint16 -func UInt16s(key string, value []uint16) Field { - return Array(key, value) -} - -// UInt32 constructs Field from uint32 -func UInt32(key string, value uint32) Field { - return UInt64(key, uint64(value)) -} - -// UInt32s constructs Field from []uint32 -func UInt32s(key string, value []uint32) Field { - return Array(key, value) -} - -// UInt64 constructs Field from uint64 -func UInt64(key string, value uint64) Field { - return Field{key: key, ftype: FieldTypeUnsigned, unsigned: value} -} - -// UInt64s constructs Field from []uint64 -func UInt64s(key string, value []uint64) Field { - return Array(key, value) -} - -// Float32 constructs Field from float32 -func Float32(key string, value float32) Field { - return Float64(key, float64(value)) -} - -// Float32s constructs Field from []float32 -func Float32s(key string, value []float32) Field { - return Array(key, value) -} - -// Float64 constructs Field from float64 -func Float64(key string, value float64) Field { - return Field{key: key, ftype: FieldTypeFloat, float: value} -} - -// Float64s constructs Field from []float64 -func Float64s(key string, value []float64) Field { - return Array(key, value) -} - -// Time constructs field of time.Time type -func Time(key string, value time.Time) Field { - return Field{key: key, ftype: FieldTypeTime, signed: value.UnixNano()} -} - -// Times constructs Field from []time.Time -func Times(key string, value []time.Time) Field { - return Array(key, value) -} - -// Duration constructs field of time.Duration type -func Duration(key string, value time.Duration) Field { - return Field{key: key, ftype: FieldTypeDuration, signed: value.Nanoseconds()} -} - -// Durations constructs Field from []time.Duration -func Durations(key string, value []time.Duration) Field { - return Array(key, value) -} - -// NamedError constructs field of error type -func NamedError(key string, value error) Field { - return Field{key: key, ftype: FieldTypeError, iface: value} -} - -// Error constructs field of error type with default field name -func Error(value error) Field { - return NamedError(DefaultErrorFieldName, value) -} - -// Errors constructs Field from []error -func Errors(key string, value []error) Field { - return Array(key, value) -} - -// Array constructs field of array type -func Array(key string, value interface{}) Field { - return Field{key: key, ftype: FieldTypeArray, iface: value} -} - -// Reflect constructs field of unknown type -func Reflect(key string, value interface{}) Field { - return Field{key: key, ftype: FieldTypeReflect, iface: value} -} - -// ByteString constructs field of bytes that could represent UTF-8 string -func ByteString(key string, value []byte) Field { - return Field{key: key, ftype: FieldTypeByteString, iface: value} -} - -// Context constructs field for lazy context fields evaluation if possible -func Context(ctx context.Context) Field { - return Field{ftype: FieldTypeContext, iface: ctx} -} - -// Any tries to deduce interface{} underlying type and constructs Field from it. -// Use of this function is ok only for the sole purpose of not repeating its entire code -// or parts of it in user's code (when you need to log interface{} types with unknown content). -// Otherwise please use specialized functions. -// nolint: gocyclo -func Any(key string, value interface{}) Field { - switch val := value.(type) { - case bool: - return Bool(key, val) - case float64: - return Float64(key, val) - case float32: - return Float32(key, val) - case int: - return Int(key, val) - case []int: - return Ints(key, val) - case int64: - return Int64(key, val) - case []int64: - return Int64s(key, val) - case int32: - return Int32(key, val) - case []int32: - return Int32s(key, val) - case int16: - return Int16(key, val) - case []int16: - return Int16s(key, val) - case int8: - return Int8(key, val) - case []int8: - return Int8s(key, val) - case string: - return String(key, val) - case []string: - return Strings(key, val) - case uint: - return UInt(key, val) - case []uint: - return UInts(key, val) - case uint64: - return UInt64(key, val) - case []uint64: - return UInt64s(key, val) - case uint32: - return UInt32(key, val) - case []uint32: - return UInt32s(key, val) - case uint16: - return UInt16(key, val) - case []uint16: - return UInt16s(key, val) - case uint8: - return UInt8(key, val) - case []byte: - return Binary(key, val) - case time.Time: - return Time(key, val) - case []time.Time: - return Times(key, val) - case time.Duration: - return Duration(key, val) - case []time.Duration: - return Durations(key, val) - case error: - return NamedError(key, val) - case []error: - return Errors(key, val) - case context.Context: - return Context(val) - default: - return Field{key: key, ftype: FieldTypeAny, iface: value} - } -} diff --git a/library/go/core/log/fields_test.go b/library/go/core/log/fields_test.go deleted file mode 100644 index ff6890b4..00000000 --- a/library/go/core/log/fields_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package log - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -// Simple test, that all type of fields are correctly zapified. -// Maybe we also need some test that checks resulting zap.Field type also. -func TestFieldAny(t *testing.T) { - for typ := FieldType(0); typ <= FieldTypeReflect; typ++ { - field := Field{ftype: typ} - assert.NotPanics(t, func() { - field.Any() - }) - } -} - -func TestAny(t *testing.T) { - var v struct{ A int } - field := Any("test", &v) - assert.Equal(t, field.ftype, FieldTypeAny) -} - -func TestReflect(t *testing.T) { - field := Reflect("test", 1) - assert.Equal(t, field.ftype, FieldTypeReflect) -} - -// TODO: test fields -// TODO: test field converters diff --git a/library/go/core/log/levels.go b/library/go/core/log/levels.go deleted file mode 100644 index 54810410..00000000 --- a/library/go/core/log/levels.go +++ /dev/null @@ -1,108 +0,0 @@ -package log - -import ( - "fmt" - "strings" -) - -// Level of logging -type Level int - -// MarshalText marshals level to text -func (l Level) MarshalText() ([]byte, error) { - if l >= maxLevel || l < 0 { - return nil, fmt.Errorf("failed to marshal log level: level value (%d) is not in the allowed range (0-%d)", l, maxLevel-1) - } - return []byte(l.String()), nil -} - -// UnmarshalText unmarshals level from text -func (l *Level) UnmarshalText(text []byte) error { - level, err := ParseLevel(string(text)) - if err != nil { - return err - } - - *l = level - return nil -} - -// Standard log levels -const ( - TraceLevel Level = iota - DebugLevel - InfoLevel - WarnLevel - ErrorLevel - FatalLevel - maxLevel -) - -func Levels() (l []Level) { - for i := 0; i < int(maxLevel); i++ { - l = append(l, Level(i)) - } - return -} - -// String values for standard log levels -const ( - TraceString = "trace" - DebugString = "debug" - InfoString = "info" - WarnString = "warn" - ErrorString = "error" - FatalString = "fatal" -) - -// String implements Stringer interface for Level -func (l Level) String() string { - switch l { - case TraceLevel: - return TraceString - case DebugLevel: - return DebugString - case InfoLevel: - return InfoString - case WarnLevel: - return WarnString - case ErrorLevel: - return ErrorString - case FatalLevel: - return FatalString - default: - // For when new log level is not added to this func (most likely never). - panic(fmt.Sprintf("unknown log level: %d", l)) - } -} - -// Set implements flag.Value interface -func (l *Level) Set(v string) error { - lvl, err := ParseLevel(v) - if err != nil { - return err - } - - *l = lvl - return nil -} - -// ParseLevel parses log level from string. Returns ErrUnknownLevel for unknown log level. -func ParseLevel(l string) (Level, error) { - switch strings.ToLower(l) { - case TraceString: - return TraceLevel, nil - case DebugString: - return DebugLevel, nil - case InfoString: - return InfoLevel, nil - case WarnString: - return WarnLevel, nil - case ErrorString: - return ErrorLevel, nil - case FatalString: - return FatalLevel, nil - default: - return FatalLevel, fmt.Errorf("unknown log level: %s", l) - } -} diff --git a/library/go/core/log/levels_test.go b/library/go/core/log/levels_test.go deleted file mode 100644 index 696b9c91..00000000 --- a/library/go/core/log/levels_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package log_test - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" -) - -var levelsToTest = []struct { - name string - level log.Level -}{ - {name: log.TraceString, level: log.TraceLevel}, - {name: log.DebugString, level: log.DebugLevel}, - {name: log.InfoString, level: log.InfoLevel}, - {name: log.WarnString, level: log.WarnLevel}, - {name: log.ErrorString, level: log.ErrorLevel}, - {name: log.FatalString, level: log.FatalLevel}, -} - -func TestLevels(t *testing.T) { - for _, levelInput := range levelsToTest { - t.Run("Convert "+levelInput.name, func(t *testing.T) { - levelFromLevelString, err := log.ParseLevel(levelInput.name) - require.NoError(t, err) - require.Equal(t, levelInput.level, levelFromLevelString) - - levelStringFromLevel := levelInput.level.String() - require.Equal(t, levelInput.name, levelStringFromLevel) - - levelFromLevelStringFromLevel, err := log.ParseLevel(levelStringFromLevel) - require.NoError(t, err) - require.Equal(t, levelInput.level, levelFromLevelStringFromLevel) - }) - } -} - -func TestLevel_MarshalText(t *testing.T) { - level := log.DebugLevel - _, err := level.MarshalText() - require.NoError(t, err) - - level = log.Level(100500) - _, err = level.MarshalText() - require.Error(t, err) - - level = log.Level(-1) - _, err = level.MarshalText() - require.Error(t, err) -} diff --git a/library/go/core/log/log.go b/library/go/core/log/log.go deleted file mode 100644 index 3e1f76e8..00000000 --- a/library/go/core/log/log.go +++ /dev/null @@ -1,134 +0,0 @@ -package log - -import "errors" - -// Logger is the universal logger that can do everything. -type Logger interface { - loggerStructured - loggerFmt - toStructured - toFmt - withName -} - -type withName interface { - WithName(name string) Logger -} - -type toLogger interface { - // Logger returns general logger - Logger() Logger -} - -// Structured provides interface for logging using fields. -type Structured interface { - loggerStructured - toFmt - toLogger -} - -type loggerStructured interface { - // Trace logs at Trace log level using fields - Trace(msg string, fields ...Field) - // Debug logs at Debug log level using fields - Debug(msg string, fields ...Field) - // Info logs at Info log level using fields - Info(msg string, fields ...Field) - // Warn logs at Warn log level using fields - Warn(msg string, fields ...Field) - // Error logs at Error log level using fields - Error(msg string, fields ...Field) - // Fatal logs at Fatal log level using fields - Fatal(msg string, fields ...Field) -} - -type toFmt interface { - // Fmt returns fmt logger - Fmt() Fmt -} - -// Fmt provides interface for logging using fmt formatter. -type Fmt interface { - loggerFmt - toStructured - toLogger -} - -type loggerFmt interface { - // Tracef logs at Trace log level using fmt formatter - Tracef(format string, args ...interface{}) - // Debugf logs at Debug log level using fmt formatter - Debugf(format string, args ...interface{}) - // Infof logs at Info log level using fmt formatter - Infof(format string, args ...interface{}) - // Warnf logs at Warn log level using fmt formatter - Warnf(format string, args ...interface{}) - // Errorf logs at Error log level using fmt formatter - Errorf(format string, args ...interface{}) - // Fatalf logs at Fatal log level using fmt formatter - Fatalf(format string, args ...interface{}) -} - -type toStructured interface { - // Structured returns structured logger - Structured() Structured -} - -// LoggerWith is an interface for 'With' function -// LoggerWith provides interface for logger modifications. -type LoggerWith interface { - // With implements 'With' - With(fields ...Field) Logger -} - -// With for loggers that implement LoggerWith interface, returns logger that -// always adds provided key/value to every log entry. Otherwise returns same logger. -func With(l Logger, fields ...Field) Logger { - e, ok := l.(LoggerWith) - if !ok { - return l - } - - return e.With(fields...) -} - -// LoggerAddCallerSkip is an interface for 'AddCallerSkip' function -type LoggerAddCallerSkip interface { - // AddCallerSkip implements 'AddCallerSkip' - AddCallerSkip(skip int) Logger -} - -// AddCallerSkip for loggers that implement LoggerAddCallerSkip interface, returns logger that -// adds caller skip to each log entry. Otherwise returns same logger. -func AddCallerSkip(l Logger, skip int) Logger { - e, ok := l.(LoggerAddCallerSkip) - if !ok { - return l - } - - return e.AddCallerSkip(skip) -} - -// WriteAt is a helper method that checks logger and writes message at given level -func WriteAt(l Structured, lvl Level, msg string, fields ...Field) error { - if l == nil { - return errors.New("nil logger given") - } - - switch lvl { - case DebugLevel: - l.Debug(msg, fields...) - case TraceLevel: - l.Trace(msg, fields...) - case InfoLevel: - l.Info(msg, fields...) - case WarnLevel: - l.Warn(msg, fields...) - case ErrorLevel: - l.Error(msg, fields...) - case FatalLevel: - l.Fatal(msg, fields...) - } - - return nil -} diff --git a/library/go/core/log/nop/nop.go b/library/go/core/log/nop/nop.go deleted file mode 100644 index f72d8c25..00000000 --- a/library/go/core/log/nop/nop.go +++ /dev/null @@ -1,73 +0,0 @@ -package nop - -import ( - "os" - - "github.com/ydb-platform/fq-connector-go/library/go/core/log" -) - -// Logger that does nothing -type Logger struct{} - -var _ log.Logger = &Logger{} -var _ log.Structured = &Logger{} -var _ log.Fmt = &Logger{} - -// Logger returns general logger -func (l *Logger) Logger() log.Logger { - return l -} - -// Fmt returns fmt logger -func (l *Logger) Fmt() log.Fmt { - return l -} - -// Structured returns structured logger -func (l *Logger) Structured() log.Structured { - return l -} - -// Trace implements Trace method of log.Logger interface -func (l *Logger) Trace(msg string, fields ...log.Field) {} - -// Tracef implements Tracef method of log.Logger interface -func (l *Logger) Tracef(format string, args ...interface{}) {} - -// Debug implements Debug method of log.Logger interface -func (l *Logger) Debug(msg string, fields ...log.Field) {} - -// Debugf implements Debugf method of log.Logger interface -func (l *Logger) Debugf(format string, args ...interface{}) {} - -// Info implements Info method of log.Logger interface -func (l *Logger) Info(msg string, fields ...log.Field) {} - -// Infof implements Infof method of log.Logger interface -func (l *Logger) Infof(format string, args ...interface{}) {} - -// Warn implements Warn method of log.Logger interface -func (l *Logger) Warn(msg string, fields ...log.Field) {} - -// Warnf implements Warnf method of log.Logger interface -func (l *Logger) Warnf(format string, args ...interface{}) {} - -// Error implements Error method of log.Logger interface -func (l *Logger) Error(msg string, fields ...log.Field) {} - -// Errorf implements Errorf method of log.Logger interface -func (l *Logger) Errorf(format string, args ...interface{}) {} - -// Fatal implements Fatal method of log.Logger interface -func (l *Logger) Fatal(msg string, fields ...log.Field) { - os.Exit(1) -} - -// Fatalf implements Fatalf method of log.Logger interface -func (l *Logger) Fatalf(format string, args ...interface{}) { - os.Exit(1) -} - -func (l *Logger) WithName(name string) log.Logger { - return l -} diff --git a/library/go/core/log/test/ctx_bench_test.go b/library/go/core/log/test/ctx_bench_test.go deleted file mode 100644 index 7468e7c6..00000000 --- a/library/go/core/log/test/ctx_bench_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package test - -import ( - "context" - "fmt" - "testing" - - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "github.com/ydb-platform/fq-connector-go/library/go/core/log/ctxlog" - "github.com/ydb-platform/fq-connector-go/library/go/core/log/zap" - "github.com/ydb-platform/fq-connector-go/library/go/core/log/zap/encoders" - uberzap "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -type nopWriteSyncer struct{} - -func (nws *nopWriteSyncer) Write(p []byte) (n int, err error) { - return len(p), nil -} - -func (nws *nopWriteSyncer) Sync() error { - return nil -} - -type levelEnabler struct { - enabled bool -} - -func (l *levelEnabler) Enabled(zapcore.Level) bool { - return l.enabled -} - -func BenchmarkWithFields(b *testing.B) { - cfg := uberzap.NewDevelopmentEncoderConfig() - cfg.TimeKey = "" - cfg.LevelKey = "" - kvEnc, _ := encoders.NewKVEncoder(cfg) - enabler := &levelEnabler{} - core := zapcore.NewCore(kvEnc, &nopWriteSyncer{}, enabler) - logger := uberzap.New(core) - ctx := ctxlog.WithFields(context.Background(), - log.String("foo", "bar"), - log.String("bar", "baz"), - ) - - l := &zap.Logger{L: logger} - for _, enabled := range []bool{true, false} { - enabler.enabled = enabled - b.Run(fmt.Sprintf("zap_%t", enabled), func(b *testing.B) { - b.ReportAllocs() - for i := 0; i < b.N; i++ { - logger.Info("test", uberzap.String("foo", "bar"), uberzap.String("bar", "baz")) - } - }) - b.Run(fmt.Sprintf("core_log_%t", enabled), func(b *testing.B) { - b.ReportAllocs() - for i := 0; i < b.N; i++ { - ctxlog.Info(ctx, l, "test", log.String("foo", "bar"), log.String("bar", "baz")) - } - }) - b.Run(fmt.Sprintf("core_log_new_%t", enabled), func(b *testing.B) { - b.ReportAllocs() - for i := 0; i < b.N; i++ { - l.Info("test", log.String("foo", "bar"), log.String("bar", "baz"), log.Context(ctx)) - } - }) - b.Run(fmt.Sprintf("core_log_fmt_%t", enabled), func(b *testing.B) { - b.ReportAllocs() - for i := 0; i < b.N; i++ { - ctxlog.Infof(ctx, l, "test %s %d", "test", 42) - } - }) - } - -} diff --git a/library/go/core/log/test/log_bench_test.go b/library/go/core/log/test/log_bench_test.go deleted file mode 100644 index a84382e5..00000000 --- a/library/go/core/log/test/log_bench_test.go +++ /dev/null @@ -1,39 +0,0 @@ -package test - -import ( - "fmt" - "testing" - - "github.com/ydb-platform/fq-connector-go/library/go/core/log" -) - -func BenchmarkOutput(b *testing.B) { - for _, loggerInput := range loggersToTest { - for _, count := range []int{0, 1, 2, 5} { - logger, err := loggerInput.factory(log.DebugLevel) - if err != nil { - b.Fatalf("failed to create logger: %s", b.Name()) - } - b.Run(fmt.Sprintf("%s fields %d", loggerInput.name, count), func(b *testing.B) { - benchmarkFields(b, logger, count) - }) - } - } -} - -func benchmarkFields(b *testing.B, logger log.Logger, count int) { - flds := genFields(count) - - for n := 0; n < b.N; n++ { - logger.Debug(msg, flds...) - } -} - -func genFields(count int) []log.Field { - flds := make([]log.Field, 0, count) - for ; count > 0; count-- { - flds = append(flds, log.String(key, value)) - } - - return flds -} diff --git a/library/go/core/log/test/log_test.go b/library/go/core/log/test/log_test.go deleted file mode 100644 index 92d0e865..00000000 --- a/library/go/core/log/test/log_test.go +++ /dev/null @@ -1,120 +0,0 @@ -package test - -import ( - "testing" - - "github.com/stretchr/testify/require" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "github.com/ydb-platform/fq-connector-go/library/go/core/log/nop" - "github.com/ydb-platform/fq-connector-go/library/go/core/log/zap" - uzap "go.uber.org/zap" -) - -var ( - msg = "msg" - msgfmt = "%s %s" - msgfmtargs = []interface{}{"hello", "world"} - key = "key" - value = "value" - withKey = "withKey" - withValue = "withValue" -) - -var loggersToTest = []struct { - name string - factory func(level log.Level) (log.Logger, error) -}{ - { - name: "Zap", - factory: func(level log.Level) (log.Logger, error) { - cfg := zap.JSONConfig(level) - // Disable output - cfg.OutputPaths = []string{} - cfg.ErrorOutputPaths = []string{} - return zap.New(cfg) - }, - }, - { - name: "ZapNop", - factory: func(level log.Level) (log.Logger, error) { - return &zap.Logger{ - L: uzap.NewNop(), - }, nil - }, - }, - { - name: "Nop", - factory: func(level log.Level) (log.Logger, error) { - return &nop.Logger{}, nil - }, - }, -} - -func TestLoggers(t *testing.T) { - for _, loggerInput := range loggersToTest { - for _, level := range log.Levels() { - t.Run("Construct "+loggerInput.name+level.String(), func(t *testing.T) { - logger, err := loggerInput.factory(level) - require.NoError(t, err) - require.NotNil(t, logger) - - lfmt := logger.Fmt() - require.NotNil(t, lfmt) - - l := lfmt.Structured() - require.NotNil(t, l) - require.Equal(t, logger, l) - }) - - t.Run("With "+loggerInput.name+level.String(), func(t *testing.T) { - logger, err := loggerInput.factory(level) - require.NoError(t, err) - require.NotNil(t, logger) - - withField := log.String(withKey, withValue) - loggerWith := log.With(logger, withField) - require.NotNil(t, loggerWith) - }) - - t.Run("AddCallerSkip "+loggerInput.name+level.String(), func(t *testing.T) { - logger, err := loggerInput.factory(level) - require.NoError(t, err) - require.NotNil(t, logger) - - loggerCallerSkip := log.AddCallerSkip(logger, 1) - require.NotNil(t, loggerCallerSkip) - }) - - // TODO: validate log output - t.Run("Logger "+loggerInput.name+level.String(), func(t *testing.T) { - logger, err := loggerInput.factory(level) - require.NoError(t, err) - require.NotNil(t, logger) - - logger.Trace(msg, log.String(key, value)) - logger.Debug(msg, log.String(key, value)) - logger.Info(msg, log.String(key, value)) - logger.Warn(msg, log.String(key, value)) - logger.Error(msg, log.String(key, value)) - // TODO: test fatal - }) - - // TODO: validate log output - t.Run("LoggerFMT "+loggerInput.name+level.String(), func(t *testing.T) { - logger, err := loggerInput.factory(level) - require.NoError(t, err) - require.NotNil(t, logger) - - lfmt := logger.Fmt() - require.NotNil(t, lfmt) - - lfmt.Tracef(msgfmt, msgfmtargs...) - lfmt.Debugf(msgfmt, msgfmtargs...) - lfmt.Infof(msgfmt, msgfmtargs...) - lfmt.Warnf(msgfmt, msgfmtargs...) - lfmt.Errorf(msgfmt, msgfmtargs...) - // TODO: test fatal - }) - } - } -} diff --git a/library/go/core/log/zap/asynczap/background.go b/library/go/core/log/zap/asynczap/background.go deleted file mode 100644 index 5af635df..00000000 --- a/library/go/core/log/zap/asynczap/background.go +++ /dev/null @@ -1,155 +0,0 @@ -package asynczap - -import ( - "bytes" - "errors" - "sync" - "sync/atomic" - "time" - - "go.uber.org/zap/buffer" - "go.uber.org/zap/zapcore" -) - -// background is a single object shared by all clones of core. -type background struct { - options Options - - q queue - out zapcore.WriteSyncer - - // use manual buffering instead of bufio background to preserve write atomicity. - // - // bufio.Writer might split log lines at arbitrary position. - writeBuffer bytes.Buffer - - wg sync.WaitGroup - mu sync.Mutex - cond *sync.Cond - stopped bool - iter int64 - lastErr error - forceFlush chan struct{} - - droppedRecords int64 - writeErrors int64 - reportOverflow int64 -} - -func newBackground(options Options, out zapcore.WriteSyncer) *background { - b := &background{ - options: options, - out: out, - forceFlush: make(chan struct{}), - } - b.cond = sync.NewCond(&b.mu) - return b -} - -func (b *background) flush() { - _, err := b.out.Write(b.writeBuffer.Bytes()) - if err != nil { - b.onError(err) - } - b.writeBuffer.Reset() -} - -func (b *background) onError(err error) { - atomic.AddInt64(&b.writeErrors, 1) - - b.lastErr = err -} - -func (b *background) stop() { - b.mu.Lock() - b.stopped = true - b.mu.Unlock() - - b.wg.Wait() -} - -func (b *background) finishIter() (stop bool) { - b.mu.Lock() - stop = b.stopped - b.mu.Unlock() - - atomic.StoreInt64(&b.reportOverflow, 0) - b.cond.Broadcast() - return -} - -func (b *background) run() { - defer b.wg.Done() - - flush := time.NewTicker(b.options.FlushInterval) - defer flush.Stop() - - var bufs []*buffer.Buffer - for { - bufs = bufs[:0] - b.mu.Lock() - - bufs = b.q.dequeueAll(bufs) - for _, buf := range bufs { - b.writeBuffer.Write(buf.Bytes()) - buf.Free() - - if b.writeBuffer.Len() > b.options.WriteBufferSize { - b.flush() - } - } - - if b.writeBuffer.Len() != 0 { - b.flush() - } - - b.iter++ - b.mu.Unlock() - - if b.finishIter() { - return - } - - select { - case <-flush.C: - case <-b.forceFlush: - flush.Reset(b.options.FlushInterval) - } - - } -} - -func (b *background) checkQueueSize() (size int, ok, shouldReport bool) { - size = int(b.q.loadSize()) - if size >= b.options.MaxMemoryUsage { - atomic.AddInt64(&b.droppedRecords, 1) - - old := atomic.SwapInt64(&b.reportOverflow, 1) - return size, false, old == 0 - } - - return 0, true, false -} - -func (b *background) sync() error { - b.mu.Lock() - defer b.mu.Unlock() - - select { - case b.forceFlush <- struct{}{}: - default: - } - - now := b.iter - for { - if b.iter >= now+1 { - return b.lastErr - } - - if b.stopped { - return errors.New("core has stopped") - } - - b.cond.Wait() - } -} diff --git a/library/go/core/log/zap/asynczap/core.go b/library/go/core/log/zap/asynczap/core.go deleted file mode 100644 index 11acd24f..00000000 --- a/library/go/core/log/zap/asynczap/core.go +++ /dev/null @@ -1,113 +0,0 @@ -// Package asynczap implements asynchronous core for zap. -// -// By default, zap writes every log line synchronously and without buffering. This behaviour -// is completely inadequate for high-rate logging. -// -// This implementation of zap.Core moves file write to background goroutine, while carefully -// monitoring memory consumption. -// -// When background goroutine can't keep up with logging rate, log records are dropped. -package asynczap - -import ( - "fmt" - "sync/atomic" - - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -type ( - Core struct { - zapcore.LevelEnabler - enc zapcore.Encoder - w *background - options Options - } - - Stats struct { - // Number of records dropped during memory overflow. - DroppedRecords int - - // Number of errors returned from underlying writer. - WriteErrors int - } -) - -// NewCore creates a Core that writes logs to a WriteSyncer. -func NewCore(enc zapcore.Encoder, ws zapcore.WriteSyncer, enab zapcore.LevelEnabler, options Options) *Core { - options.setDefault() - - w := newBackground(options, ws) - w.wg.Add(1) - go w.run() - - return &Core{ - LevelEnabler: enab, - enc: enc, - w: w, - } -} - -func (c *Core) Stop() { - _ = c.Sync() - c.w.stop() -} - -func (c *Core) With(fields []zap.Field) zapcore.Core { - clone := c.clone() - for i := range fields { - fields[i].AddTo(clone.enc) - } - return clone -} - -func (c *Core) Check(ent zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { - if c.Enabled(ent.Level) { - return ce.AddCore(ent, c) - } - return ce -} - -func (c *Core) Write(ent zapcore.Entry, fields []zap.Field) error { - if size, ok, shouldReport := c.w.checkQueueSize(); !ok { - if shouldReport { - // Report overflow error only once per background iteration, to avoid spamming error output. - return fmt.Errorf("logger queue overflow: %d >= %d", size, c.options.MaxMemoryUsage) - } else { - return nil - } - } - - buf, err := c.enc.EncodeEntry(ent, fields) - if err != nil { - return err - } - - c.w.q.enqueue(buf) - if ent.Level > zap.ErrorLevel { - // Since we may be crashing the program, sync the output. - _ = c.Sync() - } - return nil -} - -func (c *Core) Sync() error { - return c.w.sync() -} - -func (c *Core) Stat() Stats { - return Stats{ - DroppedRecords: int(atomic.LoadInt64(&c.w.droppedRecords)), - WriteErrors: int(atomic.LoadInt64(&c.w.writeErrors)), - } -} - -func (c *Core) clone() *Core { - return &Core{ - LevelEnabler: c.LevelEnabler, - enc: c.enc.Clone(), - w: c.w, - options: c.options, - } -} diff --git a/library/go/core/log/zap/asynczap/core_test.go b/library/go/core/log/zap/asynczap/core_test.go deleted file mode 100644 index 35ae2456..00000000 --- a/library/go/core/log/zap/asynczap/core_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package asynczap - -import ( - "bytes" - "os" - "runtime" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -func TestCompareToDefault(t *testing.T) { - var buf0, buf1 bytes.Buffer - out0 := zapcore.AddSync(&buf0) - out1 := zapcore.AddSync(&buf1) - - format := zap.NewProductionEncoderConfig() - format.EncodeTime = func(t time.Time, e zapcore.PrimitiveArrayEncoder) { - e.AppendString("10:00") - } - - asyncCore := NewCore( - zapcore.NewJSONEncoder(format), - out0, - zap.DebugLevel, - Options{}) - - log0 := zap.New(asyncCore) - log0.Error("foo") - - require.NoError(t, asyncCore.Sync()) - asyncCore.Stop() - - syncCore := zapcore.NewCore( - zapcore.NewJSONEncoder(format), - out1, - zap.DebugLevel) - - log1 := zap.New(syncCore) - log1.Error("foo") - - require.Equal(t, buf0.String(), buf1.String()) -} - -type countWriteSyncer int32 - -func (c *countWriteSyncer) Write(b []byte) (int, error) { - atomic.AddInt32((*int32)(c), 1) - return len(b), nil -} - -func (c *countWriteSyncer) Sync() error { - return nil -} - -func TestSync(t *testing.T) { - var c countWriteSyncer - out0 := &c - - format := zap.NewProductionEncoderConfig() - format.EncodeTime = func(t time.Time, e zapcore.PrimitiveArrayEncoder) { - e.AppendString("10:00") - } - - asyncCore := NewCore( - zapcore.NewJSONEncoder(format), - out0, - zap.DebugLevel, - Options{FlushInterval: 10 * time.Nanosecond}) - - log0 := zap.New(asyncCore) - - for i := 0; i < 100000; i++ { - log0.Error("123") - _ = log0.Sync() - require.EqualValues(t, i+1, atomic.LoadInt32((*int32)(&c))) - } -} - -type lockWriter struct { - c chan struct{} -} - -func (w *lockWriter) Write(b []byte) (int, error) { - <-w.c - return 0, nil -} - -func TestDropsRecordsOnOverflow(t *testing.T) { - go func() { - time.Sleep(time.Second * 15) - - buf := make([]byte, 1024*1024) - n := runtime.Stack(buf, true) - _, _ = os.Stderr.Write(buf[:n]) - }() - - w := &lockWriter{c: make(chan struct{})} - - asyncCore := NewCore( - zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), - zapcore.AddSync(w), - zap.DebugLevel, - Options{ - MaxMemoryUsage: 100, - }) - defer asyncCore.Stop() - - log := zap.New(asyncCore) - - for i := 0; i < 1000; i++ { - log.Error("foobar", zap.String("key", strings.Repeat("x", 1000))) - } - - assert.Greater(t, asyncCore.Stat().DroppedRecords, 990) - close(w.c) -} diff --git a/library/go/core/log/zap/asynczap/options.go b/library/go/core/log/zap/asynczap/options.go deleted file mode 100644 index 9b7f241f..00000000 --- a/library/go/core/log/zap/asynczap/options.go +++ /dev/null @@ -1,34 +0,0 @@ -package asynczap - -import "time" - -const ( - defaultMaxMemoryUsage = 1 << 26 // 64MB - defaultWriteBufferSize = 1 << 20 // 1MB - defaultFlushInterval = time.Millisecond * 100 -) - -type Options struct { - // MaxMemoryUsage is maximum amount of memory that will be used by in-flight log records. - MaxMemoryUsage int - - // WriteBufferSize specifies size of the buffer used for writes to underlying file. - WriteBufferSize int - - // FlushInterval specifies how often background goroutine would wake up. - FlushInterval time.Duration -} - -func (o *Options) setDefault() { - if o.MaxMemoryUsage == 0 { - o.MaxMemoryUsage = defaultMaxMemoryUsage - } - - if o.WriteBufferSize == 0 { - o.WriteBufferSize = defaultWriteBufferSize - } - - if o.FlushInterval == 0 { - o.FlushInterval = defaultFlushInterval - } -} diff --git a/library/go/core/log/zap/asynczap/queue.go b/library/go/core/log/zap/asynczap/queue.go deleted file mode 100644 index a37e87ef..00000000 --- a/library/go/core/log/zap/asynczap/queue.go +++ /dev/null @@ -1,83 +0,0 @@ -package asynczap - -import ( - "sync" - "sync/atomic" - "unsafe" - - "go.uber.org/zap/buffer" -) - -var entryPool sync.Pool - -func newEntry() *entry { - pooled := entryPool.Get() - if pooled != nil { - return pooled.(*entry) - } else { - return new(entry) - } -} - -func putEntry(e *entry) { - entryPool.Put(e) -} - -type entry struct { - next *entry - buf *buffer.Buffer -} - -type queue struct { - size int64 - head unsafe.Pointer -} - -func (q *queue) loadHead() *entry { - return (*entry)(atomic.LoadPointer(&q.head)) -} - -func (q *queue) casHead(old, new *entry) (swapped bool) { - return atomic.CompareAndSwapPointer(&q.head, unsafe.Pointer(old), unsafe.Pointer(new)) -} - -func (q *queue) swapHead() *entry { - return (*entry)(atomic.SwapPointer(&q.head, nil)) -} - -func (q *queue) loadSize() int64 { - return atomic.LoadInt64(&q.size) -} - -func (q *queue) enqueue(buf *buffer.Buffer) { - e := newEntry() - e.buf = buf - - atomic.AddInt64(&q.size, int64(buf.Cap())) - for { - e.next = q.loadHead() - if q.casHead(e.next, e) { - break - } - } -} - -func (q *queue) dequeueAll(to []*buffer.Buffer) []*buffer.Buffer { - head := q.swapHead() - - for head != nil { - atomic.AddInt64(&q.size, -int64(head.buf.Cap())) - to = append(to, head.buf) - - next := head.next - putEntry(head) - head = next - } - - for i := 0; i < len(to)/2; i++ { - j := len(to) - i - 1 - to[i], to[j] = to[j], to[i] - } - - return to -} diff --git a/library/go/core/log/zap/asynczap/queue_test.go b/library/go/core/log/zap/asynczap/queue_test.go deleted file mode 100644 index 25e9e62a..00000000 --- a/library/go/core/log/zap/asynczap/queue_test.go +++ /dev/null @@ -1,59 +0,0 @@ -package asynczap - -import ( - "runtime" - "testing" - "time" - - "github.com/stretchr/testify/require" - "go.uber.org/zap/buffer" -) - -func BenchmarkQueue(b *testing.B) { - var q queue - - go func() { - var buf []*buffer.Buffer - - for range time.Tick(10 * time.Millisecond) { - buf = q.dequeueAll(buf) - buf = buf[:0] - } - }() - - p := &buffer.Buffer{} - - b.ReportAllocs() - b.SetParallelism(runtime.NumCPU() - 1) - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - q.enqueue(p) - } - }) -} - -func TestQueue(t *testing.T) { - var b0, b1, b2, b3 *buffer.Buffer - b0 = &buffer.Buffer{} - b0.AppendString("b0") - b1 = &buffer.Buffer{} - b1.AppendString("b1") - b2 = &buffer.Buffer{} - b2.AppendString("b2") - b3 = &buffer.Buffer{} - b3.AppendString("b3") - - var q queue - q.enqueue(b0) - q.enqueue(b1) - q.enqueue(b2) - - require.Equal(t, []*buffer.Buffer{b0, b1, b2}, q.dequeueAll(nil)) - - q.enqueue(b0) - q.enqueue(b1) - q.enqueue(b2) - q.enqueue(b3) - - require.Equal(t, []*buffer.Buffer{b0, b1, b2, b3}, q.dequeueAll(nil)) -} diff --git a/library/go/core/log/zap/benchmark_test.go b/library/go/core/log/zap/benchmark_test.go deleted file mode 100644 index e6d3a4a8..00000000 --- a/library/go/core/log/zap/benchmark_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package zap - -import ( - "errors" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "go.uber.org/zap" -) - -func BenchmarkZapLogger(b *testing.B) { - // use config for both loggers - cfg := NewDeployConfig() - cfg.OutputPaths = nil - cfg.ErrorOutputPaths = nil - - b.Run("stock", func(b *testing.B) { - for _, level := range log.Levels() { - b.Run(level.String(), func(b *testing.B) { - cfg.Level = zap.NewAtomicLevelAt(ZapifyLevel(level)) - - logger, err := cfg.Build() - require.NoError(b, err) - - funcs := []func(string, ...zap.Field){ - logger.Debug, - logger.Info, - logger.Warn, - logger.Error, - logger.Fatal, - } - - message := "test" - fields := []zap.Field{ - zap.String("test", "test"), - zap.Bool("test", true), - zap.Int("test", 42), - } - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - funcs[i%(len(funcs)-1)](message, fields...) - } - }) - } - }) - - b.Run("wrapped", func(b *testing.B) { - for _, level := range log.Levels() { - b.Run(level.String(), func(b *testing.B) { - cfg.Level = zap.NewAtomicLevelAt(ZapifyLevel(level)) - logger, err := New(cfg) - require.NoError(b, err) - - funcs := []func(string, ...log.Field){ - logger.Debug, - logger.Info, - logger.Warn, - logger.Error, - logger.Fatal, - } - - message := "test" - fields := []log.Field{ - log.String("test", "test"), - log.Bool("test", true), - log.Int("test", 42), - } - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - funcs[i%(len(funcs)-1)](message, fields...) - } - }) - } - }) -} - -func BenchmarkZapifyField(b *testing.B) { - fields := []log.Field{ - log.Nil("test"), - log.String("test", "test"), - log.Binary("test", []byte("test")), - log.Bool("test", true), - log.Int("test", 42), - log.UInt("test", 42), - log.Float64("test", 42), - log.Time("test", time.Now()), - log.Duration("test", time.Second), - log.NamedError("test", errors.New("test")), - log.Strings("test", []string{"test"}), - log.Any("test", "test"), - log.Reflect("test", "test"), - log.ByteString("test", []byte("test")), - } - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - zapifyField(fields[i%(len(fields)-1)]) - } -} - -func BenchmarkZapifyFields(b *testing.B) { - fields := []log.Field{ - log.Nil("test"), - log.String("test", "test"), - log.Binary("test", []byte("test")), - log.Bool("test", true), - log.Int("test", 42), - log.UInt("test", 42), - log.Float64("test", 42), - log.Time("test", time.Now()), - log.Duration("test", time.Second), - log.NamedError("test", errors.New("test")), - log.Strings("test", []string{"test"}), - log.Any("test", "test"), - log.Reflect("test", "test"), - log.ByteString("test", []byte("test")), - } - - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - zapifyFields(fields...) - } -} diff --git a/library/go/core/log/zap/context.go b/library/go/core/log/zap/context.go deleted file mode 100644 index 78413914..00000000 --- a/library/go/core/log/zap/context.go +++ /dev/null @@ -1,31 +0,0 @@ -package zap - -import ( - "context" - - "github.com/ydb-platform/fq-connector-go/library/go/core/log/ctxlog" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -type ctxField struct { - ctx context.Context -} - -// MarshalLogObject implements zapcore.ObjectMarshaler to append context fields directly to encoder in a lazy manner -func (c ctxField) MarshalLogObject(encoder zapcore.ObjectEncoder) error { - fields := ctxlog.ContextFields(c.ctx) - for _, f := range fields { - zapifyField(f).AddTo(encoder) - } - return nil -} - -// Context creates a log field from context - all fields bound with ctxlog.WithFields will be added. -func Context(ctx context.Context) zap.Field { - return zap.Field{ - Key: "", - Type: zapcore.InlineMarshalerType, - Interface: ctxField{ctx: ctx}, - } -} diff --git a/library/go/core/log/zap/deploy.go b/library/go/core/log/zap/deploy.go deleted file mode 100644 index eb0863ee..00000000 --- a/library/go/core/log/zap/deploy.go +++ /dev/null @@ -1,132 +0,0 @@ -package zap - -import ( - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -// NewDeployEncoderConfig returns an opinionated EncoderConfig for -// deploy environment. -func NewDeployEncoderConfig() zapcore.EncoderConfig { - return zapcore.EncoderConfig{ - MessageKey: "msg", - LevelKey: "levelStr", - StacktraceKey: "stackTrace", - TimeKey: "@timestamp", - CallerKey: "", - NameKey: "loggerName", - EncodeLevel: zapcore.CapitalLevelEncoder, - EncodeTime: zapcore.ISO8601TimeEncoder, - EncodeDuration: zapcore.StringDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - } -} - -type cfgOption func(cfg *zap.Config) - -// WithSampling sets sampling settings initial and thereafter -func WithSampling(initial int, thereafter int) cfgOption { - return cfgOption(func(cfg *zap.Config) { - cfg.Sampling = &zap.SamplingConfig{ - Initial: initial, - Thereafter: thereafter, - } - }) -} - -// SetOutputPaths sets OutputPaths (stdout by default) -func SetOutputPaths(paths []string) cfgOption { - return cfgOption(func(cfg *zap.Config) { - cfg.OutputPaths = paths - }) -} - -// WithDevelopment sets Development option of zap.Config -func WithDevelopment(enabled bool) cfgOption { - return cfgOption(func(cfg *zap.Config) { - cfg.Development = enabled - }) -} - -// WithLevel sets level of logging -func WithLevel(level log.Level) cfgOption { - return cfgOption(func(cfg *zap.Config) { - cfg.Level = zap.NewAtomicLevelAt(ZapifyLevel(level)) - }) -} - -// NewDeployConfig returns default configuration (with no sampling). -// Not recommended for production use. -func NewDeployConfig(opts ...cfgOption) zap.Config { - cfg := zap.Config{ - Level: zap.NewAtomicLevelAt(zap.DebugLevel), - Encoding: "json", - OutputPaths: []string{"stdout"}, - ErrorOutputPaths: []string{"stderr"}, - EncoderConfig: NewDeployEncoderConfig(), - } - - for _, opt := range opts { - opt(&cfg) - } - - return cfg -} - -// NewCustomDeployLogger constructs new logger by config cfg -func NewCustomDeployLogger(cfg zap.Config, opts ...zap.Option) (*Logger, error) { - zl, err := cfg.Build(opts...) - if err != nil { - return nil, err - } - - return &Logger{ - L: addDeployContext(zl).(*zap.Logger), - }, nil -} - -// NewDeployLogger constructs fully-fledged Deploy compatible logger -// based on predefined config. See https://deploy.yandex-team.ru/docs/concepts/pod/sidecars/logs/logs#format -// for more information -func NewDeployLogger(level log.Level, opts ...zap.Option) (*Logger, error) { - return NewCustomDeployLogger( - NewDeployConfig( - WithLevel(level), - ), - opts..., - ) -} - -// NewProductionDeployConfig returns configuration, suitable for production use. -// -// It uses a JSON encoder, writes to standard error, and enables sampling. -// Stacktraces are automatically included on logs of ErrorLevel and above. -func NewProductionDeployConfig() zap.Config { - return NewDeployConfig( - WithDevelopment(false), - WithSampling(100, 100), - ) -} - -// Same as NewDeployLogger, but with sampling -func NewProductionDeployLogger(level log.Level, opts ...zap.Option) (*Logger, error) { - return NewCustomDeployLogger( - NewDeployConfig( - WithLevel(level), - WithDevelopment(false), - WithSampling(100, 100), - ), - opts..., - ) -} - -func addDeployContext(i interface{}) interface{} { - switch c := i.(type) { - case *zap.Logger: - return c.With(zap.Namespace("@fields")) - case zapcore.Core: - return c.With([]zapcore.Field{zap.Namespace("@fields")}) - } - return i -} diff --git a/library/go/core/log/zap/encoders/cli.go b/library/go/core/log/zap/encoders/cli.go deleted file mode 100644 index f19d8527..00000000 --- a/library/go/core/log/zap/encoders/cli.go +++ /dev/null @@ -1,78 +0,0 @@ -package encoders - -import ( - "sync" - - "go.uber.org/zap/buffer" - "go.uber.org/zap/zapcore" -) - -const ( - // EncoderNameCli is the encoder name to use for zap config - EncoderNameCli = "cli" -) - -var cliPool = sync.Pool{New: func() interface{} { - return &cliEncoder{} -}} - -func getCliEncoder() *cliEncoder { - return cliPool.Get().(*cliEncoder) -} - -type cliEncoder struct { - *kvEncoder -} - -// NewCliEncoder constructs cli encoder -func NewCliEncoder(cfg zapcore.EncoderConfig) (zapcore.Encoder, error) { - return newCliEncoder(cfg), nil -} - -func newCliEncoder(cfg zapcore.EncoderConfig) *cliEncoder { - return &cliEncoder{ - kvEncoder: newKVEncoder(cfg), - } -} - -func (enc *cliEncoder) Clone() zapcore.Encoder { - clone := enc.clone() - _, _ = clone.buf.Write(enc.buf.Bytes()) - return clone -} - -func (enc *cliEncoder) clone() *cliEncoder { - clone := getCliEncoder() - clone.kvEncoder = getKVEncoder() - clone.cfg = enc.cfg - clone.openNamespaces = enc.openNamespaces - clone.pool = enc.pool - clone.buf = enc.pool.Get() - return clone -} - -func (enc *cliEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { - final := enc.clone() - - // Direct write because we do not want to quote message in cli mode - final.buf.AppendString(ent.Message) - - // Add any structured context. - for _, f := range fields { - f.AddTo(final) - } - - // If there's no stacktrace key, honor that; this allows users to force - // single-line output. - if ent.Stack != "" && final.cfg.StacktraceKey != "" { - final.buf.AppendByte('\n') - final.AppendString(ent.Stack) - } - - if final.cfg.LineEnding != "" { - final.AppendString(final.cfg.LineEnding) - } else { - final.AppendString(zapcore.DefaultLineEnding) - } - return final.buf, nil -} diff --git a/library/go/core/log/zap/encoders/kv.go b/library/go/core/log/zap/encoders/kv.go deleted file mode 100644 index 8fd6c607..00000000 --- a/library/go/core/log/zap/encoders/kv.go +++ /dev/null @@ -1,386 +0,0 @@ -package encoders - -import ( - "encoding/base64" - "encoding/json" - "math" - "strings" - "sync" - "time" - - "go.uber.org/zap/buffer" - "go.uber.org/zap/zapcore" -) - -const ( - // EncoderNameKV is the encoder name to use for zap config - EncoderNameKV = "kv" -) - -const ( - // We use ' for quote symbol instead of " so that it doesn't interfere with %q of fmt package - stringQuoteSymbol = '\'' - kvArraySeparator = ',' -) - -var kvPool = sync.Pool{New: func() interface{} { - return &kvEncoder{} -}} - -func getKVEncoder() *kvEncoder { - return kvPool.Get().(*kvEncoder) -} - -type kvEncoder struct { - cfg zapcore.EncoderConfig - pool buffer.Pool - buf *buffer.Buffer - openNamespaces int - - // for encoding generic values by reflection - reflectBuf *buffer.Buffer - reflectEnc *json.Encoder -} - -// NewKVEncoder constructs kv encoder -func NewKVEncoder(cfg zapcore.EncoderConfig) (zapcore.Encoder, error) { - return newKVEncoder(cfg), nil -} - -func newKVEncoder(cfg zapcore.EncoderConfig) *kvEncoder { - pool := buffer.NewPool() - return &kvEncoder{ - cfg: cfg, - pool: pool, - buf: pool.Get(), - } -} - -func (enc *kvEncoder) addElementSeparator() { - if enc.buf.Len() == 0 { - return - } - - enc.buf.AppendByte(' ') -} - -func (enc *kvEncoder) addKey(key string) { - enc.addElementSeparator() - enc.buf.AppendString(key) - enc.buf.AppendByte('=') -} - -func (enc *kvEncoder) appendFloat(val float64, bitSize int) { - enc.appendArrayItemSeparator() - switch { - case math.IsNaN(val): - enc.buf.AppendString(`"NaN"`) - case math.IsInf(val, 1): - enc.buf.AppendString(`"+Inf"`) - case math.IsInf(val, -1): - enc.buf.AppendString(`"-Inf"`) - default: - enc.buf.AppendFloat(val, bitSize) - } -} - -func (enc *kvEncoder) AddArray(key string, arr zapcore.ArrayMarshaler) error { - enc.addKey(key) - return enc.AppendArray(arr) -} - -func (enc *kvEncoder) AddObject(key string, obj zapcore.ObjectMarshaler) error { - enc.addKey(key) - return enc.AppendObject(obj) -} - -func (enc *kvEncoder) AddBinary(key string, val []byte) { - enc.AddString(key, base64.StdEncoding.EncodeToString(val)) -} - -func (enc *kvEncoder) AddByteString(key string, val []byte) { - enc.addKey(key) - enc.AppendByteString(val) -} - -func (enc *kvEncoder) AddBool(key string, val bool) { - enc.addKey(key) - enc.AppendBool(val) -} - -func (enc *kvEncoder) AddComplex128(key string, val complex128) { - enc.addKey(key) - enc.AppendComplex128(val) -} - -func (enc *kvEncoder) AddDuration(key string, val time.Duration) { - enc.addKey(key) - enc.AppendDuration(val) -} - -func (enc *kvEncoder) AddFloat64(key string, val float64) { - enc.addKey(key) - enc.AppendFloat64(val) -} - -func (enc *kvEncoder) AddInt64(key string, val int64) { - enc.addKey(key) - enc.AppendInt64(val) -} - -func (enc *kvEncoder) resetReflectBuf() { - if enc.reflectBuf == nil { - enc.reflectBuf = enc.pool.Get() - enc.reflectEnc = json.NewEncoder(enc.reflectBuf) - } else { - enc.reflectBuf.Reset() - } -} - -func (enc *kvEncoder) AddReflected(key string, obj interface{}) error { - enc.resetReflectBuf() - err := enc.reflectEnc.Encode(obj) - if err != nil { - return err - } - enc.reflectBuf.TrimNewline() - enc.addKey(key) - _, err = enc.buf.Write(enc.reflectBuf.Bytes()) - return err -} - -func (enc *kvEncoder) OpenNamespace(key string) { - enc.addKey(key) - enc.buf.AppendByte('{') - enc.openNamespaces++ -} - -func (enc *kvEncoder) AddString(key, val string) { - enc.addKey(key) - enc.AppendString(val) -} - -func (enc *kvEncoder) AddTime(key string, val time.Time) { - enc.addKey(key) - enc.AppendTime(val) -} - -func (enc *kvEncoder) AddUint64(key string, val uint64) { - enc.addKey(key) - enc.AppendUint64(val) -} - -func (enc *kvEncoder) appendArrayItemSeparator() { - last := enc.buf.Len() - 1 - if last < 0 { - return - } - - switch enc.buf.Bytes()[last] { - case '[', '{', '=': - return - default: - enc.buf.AppendByte(kvArraySeparator) - } -} - -func (enc *kvEncoder) AppendArray(arr zapcore.ArrayMarshaler) error { - enc.appendArrayItemSeparator() - enc.buf.AppendByte('[') - err := arr.MarshalLogArray(enc) - enc.buf.AppendByte(']') - return err -} - -func (enc *kvEncoder) AppendObject(obj zapcore.ObjectMarshaler) error { - enc.appendArrayItemSeparator() - enc.buf.AppendByte('{') - err := obj.MarshalLogObject(enc) - enc.buf.AppendByte('}') - return err -} - -func (enc *kvEncoder) AppendBool(val bool) { - enc.appendArrayItemSeparator() - enc.buf.AppendBool(val) -} - -func (enc *kvEncoder) AppendByteString(val []byte) { - enc.appendArrayItemSeparator() - _, _ = enc.buf.Write(val) -} - -func (enc *kvEncoder) AppendComplex128(val complex128) { - enc.appendArrayItemSeparator() - r, i := real(val), imag(val) - - enc.buf.AppendByte('"') - // Because we're always in a quoted string, we can use strconv without - // special-casing NaN and +/-Inf. - enc.buf.AppendFloat(r, 64) - enc.buf.AppendByte('+') - enc.buf.AppendFloat(i, 64) - enc.buf.AppendByte('i') - enc.buf.AppendByte('"') -} - -func (enc *kvEncoder) AppendDuration(val time.Duration) { - cur := enc.buf.Len() - enc.cfg.EncodeDuration(val, enc) - if cur == enc.buf.Len() { - // User-supplied EncodeDuration is a no-op. Fall back to nanoseconds to keep - // JSON valid. - enc.AppendInt64(int64(val)) - } -} - -func (enc *kvEncoder) AppendInt64(val int64) { - enc.appendArrayItemSeparator() - enc.buf.AppendInt(val) -} - -func (enc *kvEncoder) AppendReflected(val interface{}) error { - enc.appendArrayItemSeparator() - enc.resetReflectBuf() - err := enc.reflectEnc.Encode(val) - if err != nil { - return err - } - enc.reflectBuf.TrimNewline() - enc.addElementSeparator() - _, err = enc.buf.Write(enc.reflectBuf.Bytes()) - return err -} - -func (enc *kvEncoder) AppendString(val string) { - enc.appendArrayItemSeparator() - var quotes bool - if strings.ContainsAny(val, " =[]{}") { - quotes = true - } - - if quotes { - enc.buf.AppendByte(stringQuoteSymbol) - } - enc.buf.AppendString(val) - if quotes { - enc.buf.AppendByte(stringQuoteSymbol) - } -} - -func (enc *kvEncoder) AppendTime(val time.Time) { - cur := enc.buf.Len() - enc.cfg.EncodeTime(val, enc) - if cur == enc.buf.Len() { - // User-supplied EncodeTime is a no-op. Fall back to nanos since epoch to keep - // output JSON valid. - enc.AppendInt64(val.UnixNano()) - } -} - -func (enc *kvEncoder) AppendUint64(val uint64) { - enc.appendArrayItemSeparator() - enc.buf.AppendUint(val) -} - -func (enc *kvEncoder) AddComplex64(k string, v complex64) { enc.AddComplex128(k, complex128(v)) } -func (enc *kvEncoder) AddFloat32(k string, v float32) { enc.AddFloat64(k, float64(v)) } -func (enc *kvEncoder) AddInt(k string, v int) { enc.AddInt64(k, int64(v)) } -func (enc *kvEncoder) AddInt32(k string, v int32) { enc.AddInt64(k, int64(v)) } -func (enc *kvEncoder) AddInt16(k string, v int16) { enc.AddInt64(k, int64(v)) } -func (enc *kvEncoder) AddInt8(k string, v int8) { enc.AddInt64(k, int64(v)) } -func (enc *kvEncoder) AddUint(k string, v uint) { enc.AddUint64(k, uint64(v)) } -func (enc *kvEncoder) AddUint32(k string, v uint32) { enc.AddUint64(k, uint64(v)) } -func (enc *kvEncoder) AddUint16(k string, v uint16) { enc.AddUint64(k, uint64(v)) } -func (enc *kvEncoder) AddUint8(k string, v uint8) { enc.AddUint64(k, uint64(v)) } -func (enc *kvEncoder) AddUintptr(k string, v uintptr) { enc.AddUint64(k, uint64(v)) } -func (enc *kvEncoder) AppendComplex64(v complex64) { enc.AppendComplex128(complex128(v)) } -func (enc *kvEncoder) AppendFloat64(v float64) { enc.appendFloat(v, 64) } -func (enc *kvEncoder) AppendFloat32(v float32) { enc.appendFloat(float64(v), 32) } -func (enc *kvEncoder) AppendInt(v int) { enc.AppendInt64(int64(v)) } -func (enc *kvEncoder) AppendInt32(v int32) { enc.AppendInt64(int64(v)) } -func (enc *kvEncoder) AppendInt16(v int16) { enc.AppendInt64(int64(v)) } -func (enc *kvEncoder) AppendInt8(v int8) { enc.AppendInt64(int64(v)) } -func (enc *kvEncoder) AppendUint(v uint) { enc.AppendUint64(uint64(v)) } -func (enc *kvEncoder) AppendUint32(v uint32) { enc.AppendUint64(uint64(v)) } -func (enc *kvEncoder) AppendUint16(v uint16) { enc.AppendUint64(uint64(v)) } -func (enc *kvEncoder) AppendUint8(v uint8) { enc.AppendUint64(uint64(v)) } -func (enc *kvEncoder) AppendUintptr(v uintptr) { enc.AppendUint64(uint64(v)) } - -func (enc *kvEncoder) Clone() zapcore.Encoder { - clone := enc.clone() - _, _ = clone.buf.Write(enc.buf.Bytes()) - return clone -} - -func (enc *kvEncoder) clone() *kvEncoder { - clone := getKVEncoder() - clone.cfg = enc.cfg - clone.openNamespaces = enc.openNamespaces - clone.pool = enc.pool - clone.buf = enc.pool.Get() - return clone -} - -// nolint: gocyclo -func (enc *kvEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { - final := enc.clone() - if final.cfg.TimeKey != "" && final.cfg.EncodeTime != nil { - final.addElementSeparator() - final.buf.AppendString(final.cfg.TimeKey + "=") - final.cfg.EncodeTime(ent.Time, final) - } - if final.cfg.LevelKey != "" && final.cfg.EncodeLevel != nil { - final.addElementSeparator() - final.buf.AppendString(final.cfg.LevelKey + "=") - final.cfg.EncodeLevel(ent.Level, final) - } - if ent.LoggerName != "" && final.cfg.NameKey != "" { - nameEncoder := final.cfg.EncodeName - - if nameEncoder == nil { - // Fall back to FullNameEncoder for backward compatibility. - nameEncoder = zapcore.FullNameEncoder - } - - final.addElementSeparator() - final.buf.AppendString(final.cfg.NameKey + "=") - nameEncoder(ent.LoggerName, final) - } - if ent.Caller.Defined && final.cfg.CallerKey != "" && final.cfg.EncodeCaller != nil { - final.addElementSeparator() - final.buf.AppendString(final.cfg.CallerKey + "=") - final.cfg.EncodeCaller(ent.Caller, final) - } - - if enc.buf.Len() > 0 { - final.addElementSeparator() - _, _ = final.buf.Write(enc.buf.Bytes()) - } - - // Add the message itself. - if final.cfg.MessageKey != "" { - final.addElementSeparator() - final.buf.AppendString(final.cfg.MessageKey + "=") - final.AppendString(ent.Message) - } - - // Add any structured context. - for _, f := range fields { - f.AddTo(final) - } - - // If there's no stacktrace key, honor that; this allows users to force - // single-line output. - if ent.Stack != "" && final.cfg.StacktraceKey != "" { - final.buf.AppendByte('\n') - final.buf.AppendString(ent.Stack) - } - - if final.cfg.LineEnding != "" { - final.buf.AppendString(final.cfg.LineEnding) - } else { - final.buf.AppendString(zapcore.DefaultLineEnding) - } - return final.buf, nil -} diff --git a/library/go/core/log/zap/encoders/kv_test.go b/library/go/core/log/zap/encoders/kv_test.go deleted file mode 100644 index 85778e88..00000000 --- a/library/go/core/log/zap/encoders/kv_test.go +++ /dev/null @@ -1,121 +0,0 @@ -package encoders - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -func TestKVEncodeEntry(t *testing.T) { - type bar struct { - Key string `json:"key"` - - Val float64 `json:"val"` - } - - type foo struct { - A string `json:"aee"` - B int `json:"bee"` - C float64 `json:"cee"` - D []bar `json:"dee"` - } - - tests := []struct { - desc string - expected string - ent zapcore.Entry - fields []zapcore.Field - }{ - { - desc: "info entry with some fields", - expected: `T=2018-06-19T16:33:42.000Z L=info N=bob M='lob law' so=passes answer=42 common_pie=3.14 ` + - `such={"aee":"lol","bee":123,"cee":0.9999,"dee":[{"key":"pi","val":3.141592653589793},` + - `{"key":"tau","val":6.283185307179586}]} -`, - ent: zapcore.Entry{ - Level: zapcore.InfoLevel, - Time: time.Date(2018, 6, 19, 16, 33, 42, 99, time.UTC), - LoggerName: "bob", - Message: "lob law", - }, - fields: []zapcore.Field{ - zap.String("so", "passes"), - zap.Int("answer", 42), - zap.Float64("common_pie", 3.14), - zap.Reflect("such", foo{ - A: "lol", - B: 123, - C: 0.9999, - D: []bar{ - {"pi", 3.141592653589793}, - {"tau", 6.283185307179586}, - }, - }), - }, - }, - { - desc: "info entry with array fields", - expected: `T=2020-06-26T11:13:42.000Z L=info N=alice M='str array' env=test ` + - `intarray=[-5,-7,0,-12] ` + - `uintarray=[1,2,3,4,5] ` + - `strarray=[funny,bunny] ` + - `book=['Alice's Adventures in Wonderland','Lewis Carroll',26-11-1865] ` + - `floatarray=[3.14,-2.17,0.0000000000000000000000000000000000662607]` + "\n", - ent: zapcore.Entry{ - Level: zapcore.InfoLevel, - Time: time.Date(2020, 6, 26, 11, 13, 42, 0, time.UTC), - LoggerName: "alice", - Message: "str array", - }, - fields: []zapcore.Field{ - zap.String("env", "test"), - zap.Ints("intarray", []int{-5, -7, 0, -12}), - zap.Uints("uintarray", []uint{1, 2, 3, 4, 5}), - zap.Strings("strarray", []string{"funny", "bunny"}), - zap.Strings("book", []string{"Alice's Adventures in Wonderland", "Lewis Carroll", "26-11-1865"}), - zap.Float32s("floatarray", []float32{3.14, -2.17, 0.662607015e-34}), - }, - }, - { - desc: "corner cases of arrays", - expected: "T=2020-06-26T12:13:42.000Z L=info N=zorg M='str array' cornerequal=['hello=',world] cornerbracket=['is[',jail,']'] cornerbraces=['is{',exit,'}']\n", - ent: zapcore.Entry{ - Level: zapcore.InfoLevel, - Time: time.Date(2020, 6, 26, 12, 13, 42, 0, time.UTC), - LoggerName: "zorg", - Message: "str array", - }, - fields: []zapcore.Field{ - zap.Strings("cornerequal", []string{"hello=", "world"}), - zap.Strings("cornerbracket", []string{"is[", "jail", "]"}), - zap.Strings("cornerbraces", []string{"is{", "exit", "}"}), - }, - }, - } - - enc, _ := NewKVEncoder(zapcore.EncoderConfig{ - MessageKey: "M", - LevelKey: "L", - TimeKey: "T", - NameKey: "N", - CallerKey: "C", - StacktraceKey: "S", - EncodeLevel: zapcore.LowercaseLevelEncoder, - EncodeTime: zapcore.ISO8601TimeEncoder, - EncodeDuration: zapcore.SecondsDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - }) - - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - buf, err := enc.EncodeEntry(tt.ent, tt.fields) - if assert.NoError(t, err, "Unexpected KV encoding error.") { - assert.Equal(t, tt.expected, buf.String(), "Incorrect encoded KV entry.") - } - buf.Free() - }) - } -} diff --git a/library/go/core/log/zap/encoders/tskv.go b/library/go/core/log/zap/encoders/tskv.go deleted file mode 100644 index 2af5e27b..00000000 --- a/library/go/core/log/zap/encoders/tskv.go +++ /dev/null @@ -1,442 +0,0 @@ -package encoders - -import ( - "encoding/hex" - "encoding/json" - "fmt" - "math" - "strings" - "sync" - "time" - - "github.com/ydb-platform/fq-connector-go/library/go/core/xerrors" - "go.uber.org/zap/buffer" - "go.uber.org/zap/zapcore" -) - -const ( - // EncoderNameKV is the encoder name to use for zap config - EncoderNameTSKV = "tskv" -) - -const ( - tskvLineEnding = '\n' - tskvElementSeparator = '\t' - tskvKVSeparator = '=' - tskvMark = "tskv" - tskvArrayStart = '[' - tskvArrayEnd = ']' - tskvArraySeparator = ',' -) - -var tskvKeyEscapeRules = []string{ - `\`, `\\`, - "\t", "\\t", - "\n", "\\n", - "\r", `\r`, - "\x00", `\0`, - "=", `\=`, -} - -var tskvValueEscapeRules = []string{ - `\`, `\\`, - "\t", "\\t", - "\n", `\n`, - "\r", `\r`, - "\x00", `\0`, -} - -type tskvEscaper struct { - keyReplacer *strings.Replacer - valueReplacer *strings.Replacer -} - -func newTSKVEscaper() tskvEscaper { - return tskvEscaper{ - keyReplacer: strings.NewReplacer(tskvKeyEscapeRules...), - valueReplacer: strings.NewReplacer(tskvValueEscapeRules...), - } -} - -func (esc *tskvEscaper) escapeKey(key string) string { - return esc.keyReplacer.Replace(key) -} - -func (esc *tskvEscaper) escapeValue(val string) string { - return esc.valueReplacer.Replace(val) -} - -func hexEncode(val []byte) []byte { - dst := make([]byte, hex.EncodedLen(len(val))) - hex.Encode(dst, val) - return dst -} - -var tskvPool = sync.Pool{New: func() interface{} { - return &tskvEncoder{} -}} - -func getTSKVEncoder() *tskvEncoder { - return tskvPool.Get().(*tskvEncoder) -} - -type tskvEncoder struct { - cfg zapcore.EncoderConfig - pool buffer.Pool - buf *buffer.Buffer - - // for encoding generic values by reflection - reflectBuf *buffer.Buffer - reflectEnc *json.Encoder - - tskvEscaper tskvEscaper -} - -// NewKVEncoder constructs tskv encoder -func NewTSKVEncoder(cfg zapcore.EncoderConfig) (zapcore.Encoder, error) { - return newTSKVEncoder(cfg), nil -} - -func newTSKVEncoder(cfg zapcore.EncoderConfig) *tskvEncoder { - pool := buffer.NewPool() - return &tskvEncoder{ - cfg: cfg, - pool: pool, - buf: pool.Get(), - tskvEscaper: newTSKVEscaper(), - } -} - -func (enc *tskvEncoder) appendElementSeparator() { - if enc.buf.Len() == 0 { - return - } - - enc.buf.AppendByte(tskvElementSeparator) -} - -func (enc *tskvEncoder) appendArrayItemSeparator() { - last := enc.buf.Len() - 1 - if last < 0 { - return - } - - switch enc.buf.Bytes()[last] { - case tskvArrayStart, tskvKVSeparator: - return - default: - enc.buf.AppendByte(tskvArraySeparator) - } -} - -func (enc *tskvEncoder) safeAppendKey(key string) { - enc.appendElementSeparator() - enc.buf.AppendString(enc.tskvEscaper.escapeKey(key)) - enc.buf.AppendByte(tskvKVSeparator) -} - -func (enc *tskvEncoder) safeAppendString(val string) { - enc.buf.AppendString(enc.tskvEscaper.escapeValue(val)) -} - -func (enc *tskvEncoder) appendFloat(val float64, bitSize int) { - enc.appendArrayItemSeparator() - switch { - case math.IsNaN(val): - enc.buf.AppendString(`"NaN"`) - case math.IsInf(val, 1): - enc.buf.AppendString(`"+Inf"`) - case math.IsInf(val, -1): - enc.buf.AppendString(`"-Inf"`) - default: - enc.buf.AppendFloat(val, bitSize) - } -} - -func (enc *tskvEncoder) AddArray(key string, arr zapcore.ArrayMarshaler) error { - enc.safeAppendKey(key) - return enc.AppendArray(arr) -} - -func (enc *tskvEncoder) AddObject(key string, obj zapcore.ObjectMarshaler) error { - enc.safeAppendKey(key) - return enc.AppendObject(obj) -} - -func (enc *tskvEncoder) AddBinary(key string, val []byte) { - enc.AddByteString(key, val) -} - -func (enc *tskvEncoder) AddByteString(key string, val []byte) { - enc.safeAppendKey(key) - enc.AppendByteString(val) -} - -func (enc *tskvEncoder) AddBool(key string, val bool) { - enc.safeAppendKey(key) - enc.AppendBool(val) -} - -func (enc *tskvEncoder) AddComplex128(key string, val complex128) { - enc.safeAppendKey(key) - enc.AppendComplex128(val) -} - -func (enc *tskvEncoder) AddDuration(key string, val time.Duration) { - enc.safeAppendKey(key) - enc.AppendDuration(val) -} - -func (enc *tskvEncoder) AddFloat64(key string, val float64) { - enc.safeAppendKey(key) - enc.AppendFloat64(val) -} - -func (enc *tskvEncoder) AddInt64(key string, val int64) { - enc.safeAppendKey(key) - enc.AppendInt64(val) -} - -func (enc *tskvEncoder) resetReflectBuf() { - if enc.reflectBuf == nil { - enc.reflectBuf = enc.pool.Get() - enc.reflectEnc = json.NewEncoder(enc.reflectBuf) - } else { - enc.reflectBuf.Reset() - } -} - -func (enc *tskvEncoder) AddReflected(key string, obj interface{}) error { - enc.resetReflectBuf() - err := enc.reflectEnc.Encode(obj) - if err != nil { - return err - } - enc.reflectBuf.TrimNewline() - enc.safeAppendKey(key) - enc.safeAppendString(enc.reflectBuf.String()) - return err -} - -// OpenNamespace is not supported due to tskv format design -// See AppendObject() for more details -func (enc *tskvEncoder) OpenNamespace(key string) { - panic("TSKV encoder does not support namespaces") -} - -func (enc *tskvEncoder) AddString(key, val string) { - enc.safeAppendKey(key) - enc.safeAppendString(val) -} - -func (enc *tskvEncoder) AddTime(key string, val time.Time) { - enc.safeAppendKey(key) - enc.AppendTime(val) -} - -func (enc *tskvEncoder) AddUint64(key string, val uint64) { - enc.safeAppendKey(key) - enc.AppendUint64(val) -} - -func (enc *tskvEncoder) AppendArray(arr zapcore.ArrayMarshaler) error { - enc.appendArrayItemSeparator() - enc.buf.AppendByte(tskvArrayStart) - err := arr.MarshalLogArray(enc) - enc.buf.AppendByte(tskvArrayEnd) - return err -} - -// TSKV format does not support hierarchy data so we can't log Objects here -// The only thing we can do is to implicitly use fmt.Stringer interface -// -// ObjectMarshaler interface requires MarshalLogObject method -// from within MarshalLogObject you only have access to ObjectEncoder methods (AddString, AddBool ...) -// so if you call AddString then object log will be split by \t sign -// but \t is key-value separator and tskv doesn't have another separators -// e.g -// json encoded: objLogFieldName={"innerObjKey1":{"innerObjKey2":"value"}} -// tskv encoded: objLogFieldName={ \tinnerObjKey1={ \tinnerObjKey2=value}} -func (enc *tskvEncoder) AppendObject(obj zapcore.ObjectMarshaler) error { - var err error - - enc.appendArrayItemSeparator() - enc.buf.AppendByte('{') - stringerObj, ok := obj.(fmt.Stringer) - if !ok { - err = xerrors.Errorf("fmt.Stringer implementation required due to marshall into tskv format") - } else { - enc.safeAppendString(stringerObj.String()) - } - enc.buf.AppendByte('}') - - return err -} - -func (enc *tskvEncoder) AppendBool(val bool) { - enc.appendArrayItemSeparator() - enc.buf.AppendBool(val) -} - -func (enc *tskvEncoder) AppendByteString(val []byte) { - enc.appendArrayItemSeparator() - _, _ = enc.buf.Write(hexEncode(val)) -} - -func (enc *tskvEncoder) AppendComplex128(val complex128) { // TODO - enc.appendArrayItemSeparator() - - r, i := real(val), imag(val) - enc.buf.AppendByte('"') - // Because we're always in a quoted string, we can use strconv without - // special-casing NaN and +/-Inf. - enc.buf.AppendFloat(r, 64) - enc.buf.AppendByte('+') - enc.buf.AppendFloat(i, 64) - enc.buf.AppendByte('i') - enc.buf.AppendByte('"') -} - -func (enc *tskvEncoder) AppendDuration(val time.Duration) { - cur := enc.buf.Len() - enc.cfg.EncodeDuration(val, enc) - if cur == enc.buf.Len() { - // User-supplied EncodeDuration is a no-op. Fall back to nanoseconds - enc.AppendInt64(int64(val)) - } -} - -func (enc *tskvEncoder) AppendInt64(val int64) { - enc.appendArrayItemSeparator() - enc.buf.AppendInt(val) -} - -func (enc *tskvEncoder) AppendReflected(val interface{}) error { - enc.appendArrayItemSeparator() - - enc.resetReflectBuf() - err := enc.reflectEnc.Encode(val) - if err != nil { - return err - } - enc.reflectBuf.TrimNewline() - enc.safeAppendString(enc.reflectBuf.String()) - return nil -} - -func (enc *tskvEncoder) AppendString(val string) { - enc.appendArrayItemSeparator() - enc.safeAppendString(val) -} - -func (enc *tskvEncoder) AppendTime(val time.Time) { - cur := enc.buf.Len() - enc.cfg.EncodeTime(val, enc) - if cur == enc.buf.Len() { - // User-supplied EncodeTime is a no-op. Fall back to nanos since epoch to keep output tskv valid. - enc.AppendInt64(val.Unix()) - } -} - -func (enc *tskvEncoder) AppendUint64(val uint64) { - enc.appendArrayItemSeparator() - enc.buf.AppendUint(val) -} - -func (enc *tskvEncoder) AddComplex64(k string, v complex64) { enc.AddComplex128(k, complex128(v)) } -func (enc *tskvEncoder) AddFloat32(k string, v float32) { enc.AddFloat64(k, float64(v)) } -func (enc *tskvEncoder) AddInt(k string, v int) { enc.AddInt64(k, int64(v)) } -func (enc *tskvEncoder) AddInt32(k string, v int32) { enc.AddInt64(k, int64(v)) } -func (enc *tskvEncoder) AddInt16(k string, v int16) { enc.AddInt64(k, int64(v)) } -func (enc *tskvEncoder) AddInt8(k string, v int8) { enc.AddInt64(k, int64(v)) } -func (enc *tskvEncoder) AddUint(k string, v uint) { enc.AddUint64(k, uint64(v)) } -func (enc *tskvEncoder) AddUint32(k string, v uint32) { enc.AddUint64(k, uint64(v)) } -func (enc *tskvEncoder) AddUint16(k string, v uint16) { enc.AddUint64(k, uint64(v)) } -func (enc *tskvEncoder) AddUint8(k string, v uint8) { enc.AddUint64(k, uint64(v)) } -func (enc *tskvEncoder) AddUintptr(k string, v uintptr) { enc.AddUint64(k, uint64(v)) } -func (enc *tskvEncoder) AppendComplex64(v complex64) { enc.AppendComplex128(complex128(v)) } -func (enc *tskvEncoder) AppendFloat64(v float64) { enc.appendFloat(v, 64) } -func (enc *tskvEncoder) AppendFloat32(v float32) { enc.appendFloat(float64(v), 32) } -func (enc *tskvEncoder) AppendInt(v int) { enc.AppendInt64(int64(v)) } -func (enc *tskvEncoder) AppendInt32(v int32) { enc.AppendInt64(int64(v)) } -func (enc *tskvEncoder) AppendInt16(v int16) { enc.AppendInt64(int64(v)) } -func (enc *tskvEncoder) AppendInt8(v int8) { enc.AppendInt64(int64(v)) } -func (enc *tskvEncoder) AppendUint(v uint) { enc.AppendUint64(uint64(v)) } -func (enc *tskvEncoder) AppendUint32(v uint32) { enc.AppendUint64(uint64(v)) } -func (enc *tskvEncoder) AppendUint16(v uint16) { enc.AppendUint64(uint64(v)) } -func (enc *tskvEncoder) AppendUint8(v uint8) { enc.AppendUint64(uint64(v)) } -func (enc *tskvEncoder) AppendUintptr(v uintptr) { enc.AppendUint64(uint64(v)) } - -func (enc *tskvEncoder) Clone() zapcore.Encoder { - clone := enc.clone() - _, _ = clone.buf.Write(enc.buf.Bytes()) - return clone -} - -func (enc *tskvEncoder) clone() *tskvEncoder { - clone := getTSKVEncoder() - clone.cfg = enc.cfg - clone.pool = enc.pool - clone.buf = enc.pool.Get() - clone.tskvEscaper = enc.tskvEscaper - return clone -} - -// nolint: gocyclo -func (enc *tskvEncoder) EncodeEntry(ent zapcore.Entry, fields []zapcore.Field) (*buffer.Buffer, error) { - final := enc.clone() - final.AppendString(tskvMark) - - if final.cfg.TimeKey != "" && final.cfg.EncodeTime != nil { - final.safeAppendKey(final.cfg.TimeKey) - final.cfg.EncodeTime(ent.Time, final) - } - if final.cfg.LevelKey != "" && final.cfg.EncodeLevel != nil { - final.safeAppendKey(final.cfg.LevelKey) - final.cfg.EncodeLevel(ent.Level, final) - } - if ent.LoggerName != "" && final.cfg.NameKey != "" { - nameEncoder := final.cfg.EncodeName - - if nameEncoder == nil { - // Fall back to FullNameEncoder for backward compatibility. - nameEncoder = zapcore.FullNameEncoder - } - - final.safeAppendKey(final.cfg.NameKey) - nameEncoder(ent.LoggerName, final) - } - if ent.Caller.Defined && final.cfg.CallerKey != "" && final.cfg.EncodeCaller != nil { - final.safeAppendKey(final.cfg.CallerKey) - final.cfg.EncodeCaller(ent.Caller, final) - } - - if enc.buf.Len() > 0 { - final.appendElementSeparator() - _, _ = final.buf.Write(enc.buf.Bytes()) - } - - // Add the message itself. - if final.cfg.MessageKey != "" { - final.safeAppendKey(final.cfg.MessageKey) - final.safeAppendString(ent.Message) - } - - // Add any structured context. - for _, f := range fields { - f.AddTo(final) - } - - if ent.Stack != "" && final.cfg.StacktraceKey != "" { - final.safeAppendKey(final.cfg.StacktraceKey) - final.safeAppendString(ent.Stack) - } - - if final.cfg.LineEnding != "" { - final.buf.AppendString(final.cfg.LineEnding) - } else { - final.buf.AppendByte(tskvLineEnding) - } - - return final.buf, nil -} diff --git a/library/go/core/log/zap/encoders/tskv_test.go b/library/go/core/log/zap/encoders/tskv_test.go deleted file mode 100644 index 44a74111..00000000 --- a/library/go/core/log/zap/encoders/tskv_test.go +++ /dev/null @@ -1,600 +0,0 @@ -package encoders - -import ( - "errors" - "fmt" - "math" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "go.uber.org/zap" - "go.uber.org/zap/buffer" - "go.uber.org/zap/zapcore" -) - -func TestTSKVEscaper(t *testing.T) { - tests := []struct { - input string - expectedKey string - expectedValue string - desc string - }{ - { - input: "plain text$ no need to escape", - expectedKey: "plain text$ no need to escape", - expectedValue: "plain text$ no need to escape", - desc: "test without escape", - }, { - input: "test escape\tab", - expectedKey: `test escape\tab`, - expectedValue: `test escape\tab`, - desc: "escape tab", - }, - { - input: "\ntest es\\cape\t\t a\rll char\x00s in string=", - expectedKey: `\ntest es\\cape\t\t a\rll char\0s in string\=`, - expectedValue: `\ntest es\\cape\t\t a\rll char\0s in string=`, - desc: "escape all chars", - }, - } - esc := newTSKVEscaper() - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - cur := esc.escapeKey(tt.input) - assert.Equal(t, tt.expectedKey, cur, "Incorrect escaped TSKV key.") - }) - - t.Run(tt.desc, func(t *testing.T) { - cur := esc.escapeValue(tt.input) - assert.Equal(t, tt.expectedValue, cur, "Incorrect escaped TSKV value.") - }) - } -} - -type noJSON struct{} - -func (nj noJSON) MarshalJSON() ([]byte, error) { - return nil, errors.New("no") -} - -type nonloggable struct{} - -func (l nonloggable) MarshalLogObject(enc zapcore.ObjectEncoder) error { - return nil -} - -type loggable struct { - bool bool - spec string -} - -func (l loggable) MarshalLogObject(enc zapcore.ObjectEncoder) error { - return nil -} - -func (l loggable) String() string { - - return fmt.Sprintf("loggable%s=%t%s", l.spec, l.bool, l.spec) -} - -func (l loggable) MarshalLogArray(enc zapcore.ArrayEncoder) error { - if !l.bool { - return errors.New("can't marshal") - } - enc.AppendBool(l.bool) - return nil -} - -type loggables int - -func (ls loggables) MarshalLogArray(enc zapcore.ArrayEncoder) error { - l := loggable{true, ""} - for i := 0; i < int(ls); i++ { - if err := enc.AppendObject(l); err != nil { - return err - } - } - return nil -} - -func getCommonTestConfig() zapcore.EncoderConfig { - return zapcore.EncoderConfig{ - MessageKey: "M", - LevelKey: "L", - TimeKey: "T", - NameKey: "N", - CallerKey: "C", - StacktraceKey: "S", - EncodeLevel: zapcore.LowercaseLevelEncoder, - EncodeTime: zapcore.EpochTimeEncoder, - EncodeDuration: zapcore.SecondsDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - } -} - -func getSpecCharsTestConfig() zapcore.EncoderConfig { - return zapcore.EncoderConfig{ - MessageKey: "M\t", - LevelKey: "L\n", - TimeKey: "T\r", - NameKey: "N\x00", - CallerKey: "C=", - StacktraceKey: "S\\", - EncodeLevel: zapcore.LowercaseLevelEncoder, - EncodeTime: zapcore.EpochTimeEncoder, - EncodeDuration: zapcore.SecondsDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - } -} - -func getRawTskvEncoder() *tskvEncoder { - pool := buffer.NewPool() - cfg := getCommonTestConfig() - return &tskvEncoder{ - cfg: cfg, - pool: pool, - buf: pool.Get(), - tskvEscaper: newTSKVEscaper(), - } - -} - -func assertOutput(t testing.TB, expected string, f func(encoder zapcore.Encoder)) { - enc := getRawTskvEncoder() - f(enc) - assert.Equal(t, expected, enc.buf.String(), "Unexpected encoder output after adding.") - - enc.buf.Reset() - enc.AddString("foo", "bar") - f(enc) - expectedPrefix := `foo=bar` - if expected != "" { - // If we expect output, it should be tab-separated from the previous - // field. - expectedPrefix += "\t" - } - assert.Equal(t, expectedPrefix+expected, enc.buf.String(), "Unexpected encoder output after adding as a second field.") -} - -func TestTSKVEncoderObjectFields(t *testing.T) { - tests := []struct { - desc string - expected string - f func(encoder zapcore.Encoder) - }{ - {"binary", `k=61623132`, func(e zapcore.Encoder) { e.AddBinary("k", []byte("ab12")) }}, - {"binary esc ", `k\n=61623132`, func(e zapcore.Encoder) { e.AddBinary("k\n", []byte("ab12")) }}, - {"bool", `k=true`, func(e zapcore.Encoder) { e.AddBool("k", true) }}, - {"bool", `k\t=false`, func(e zapcore.Encoder) { e.AddBool("k\t", false) }}, - - {"byteString", `k=765c`, func(e zapcore.Encoder) { e.AddByteString(`k`, []byte(`v\`)) }}, - {"byteString esc", `k\t=61623132`, func(e zapcore.Encoder) { e.AddByteString("k\t", []byte("ab12")) }}, - {"byteString empty val", `k=`, func(e zapcore.Encoder) { e.AddByteString("k", []byte{}) }}, - {"byteString nil val", `k=`, func(e zapcore.Encoder) { e.AddByteString("k", nil) }}, - - {"complex128", `k="1+2i"`, func(e zapcore.Encoder) { e.AddComplex128("k", 1+2i) }}, - {"complex128 esc", `k\t="1+2i"`, func(e zapcore.Encoder) { e.AddComplex128("k\t", 1+2i) }}, - {"complex64", `k="1+2i"`, func(e zapcore.Encoder) { e.AddComplex64("k", 1+2i) }}, - {"complex64 esc", `k\t="1+2i"`, func(e zapcore.Encoder) { e.AddComplex64("k\t", 1+2i) }}, - - {"duration", `k$=0.000000001`, func(e zapcore.Encoder) { e.AddDuration("k$", 1) }}, - {"duration esc", `k\t=0.000000001`, func(e zapcore.Encoder) { e.AddDuration("k\t", 1) }}, - - {"float64", `k=1`, func(e zapcore.Encoder) { e.AddFloat64("k", 1.0) }}, - {"float64 esc", `k\t=1`, func(e zapcore.Encoder) { e.AddFloat64("k\t", 1.0) }}, - {"float64", `k=10000000000`, func(e zapcore.Encoder) { e.AddFloat64("k", 1e10) }}, - {"float64", `k="NaN"`, func(e zapcore.Encoder) { e.AddFloat64("k", math.NaN()) }}, - {"float64", `k="+Inf"`, func(e zapcore.Encoder) { e.AddFloat64("k", math.Inf(1)) }}, - {"float64", `k="-Inf"`, func(e zapcore.Encoder) { e.AddFloat64("k", math.Inf(-1)) }}, - - {"float32", `k=1`, func(e zapcore.Encoder) { e.AddFloat32("k", 1.0) }}, - {"float32", `k\t=1`, func(e zapcore.Encoder) { e.AddFloat32("k\t", 1.0) }}, - {"float32", `k=10000000000`, func(e zapcore.Encoder) { e.AddFloat32("k", 1e10) }}, - {"float32", `k="NaN"`, func(e zapcore.Encoder) { e.AddFloat32("k", float32(math.NaN())) }}, - {"float32", `k="+Inf"`, func(e zapcore.Encoder) { e.AddFloat32("k", float32(math.Inf(1))) }}, - {"float32", `k="-Inf"`, func(e zapcore.Encoder) { e.AddFloat32("k", float32(math.Inf(-1))) }}, - - {"int", `k=42`, func(e zapcore.Encoder) { e.AddInt("k", 42) }}, - {"int esc", `k\t=42`, func(e zapcore.Encoder) { e.AddInt("k\t", 42) }}, - {"int64", `k=42`, func(e zapcore.Encoder) { e.AddInt64("k", 42) }}, - {"int32", `k=42`, func(e zapcore.Encoder) { e.AddInt32("k", 42) }}, - {"int16", `k=42`, func(e zapcore.Encoder) { e.AddInt16("k", 42) }}, - {"int8", `k=42`, func(e zapcore.Encoder) { e.AddInt8("k", 42) }}, - - {"string", `k=v$`, func(e zapcore.Encoder) { e.AddString("k", "v$") }}, - {"string esc", `k\t=v\\`, func(e zapcore.Encoder) { e.AddString("k\t", `v\`) }}, - {"string", `k=`, func(e zapcore.Encoder) { e.AddString("k", "") }}, - - {"time", `k=1`, func(e zapcore.Encoder) { e.AddTime("k", time.Unix(1, 0)) }}, - {"time esc", `k\t=1`, func(e zapcore.Encoder) { e.AddTime("k\t", time.Unix(1, 0)) }}, - - {"uint", `k=42`, func(e zapcore.Encoder) { e.AddUint("k", 42) }}, - {"uint esc", `k\t=42`, func(e zapcore.Encoder) { e.AddUint("k\t", 42) }}, - {"uint64", `k=42`, func(e zapcore.Encoder) { e.AddUint64("k", 42) }}, - {"uint32", `k=42`, func(e zapcore.Encoder) { e.AddUint32("k", 42) }}, - {"uint16", `k=42`, func(e zapcore.Encoder) { e.AddUint16("k", 42) }}, - {"uint8", `k=42`, func(e zapcore.Encoder) { e.AddUint8("k", 42) }}, - {"uintptr", `k=42`, func(e zapcore.Encoder) { e.AddUintptr("k", 42) }}, - { - desc: "object (success)", - expected: `k={loggable=true}`, - f: func(e zapcore.Encoder) { - assert.NoError(t, e.AddObject("k", loggable{true, ""}), "Unexpected error calling AddObject.") - }, - }, - { - desc: "object esc (success)", - expected: `k={loggable\t=true\t}`, - f: func(e zapcore.Encoder) { - assert.NoError(t, e.AddObject("k", loggable{true, "\t"}), "Unexpected error calling AddObject.") - }, - }, - { - desc: "object (error)", - expected: `k={}`, - f: func(e zapcore.Encoder) { - assert.Error(t, e.AddObject("k", nonloggable{}), "Expected an error calling AddObject.") - }, - }, - { - desc: "array (with nested object)", - expected: `loggables=[{loggable=true},{loggable=true}]`, - f: func(e zapcore.Encoder) { - assert.NoError( - t, - e.AddArray("loggables", loggables(2)), - "Unexpected error calling AddObject with nested ArrayMarshalers.", - ) - }, - }, - { - desc: "array (success)", - expected: `k=[true]`, - f: func(e zapcore.Encoder) { - assert.NoError(t, e.AddArray(`k`, loggable{true, ""}), "Unexpected error calling MarshalLogArray.") - }, - }, - { - desc: "array esc (success)", - expected: `k\t=[true]`, - f: func(e zapcore.Encoder) { - assert.NoError(t, e.AddArray("k\t", loggable{true, ""}), "Unexpected error calling MarshalLogArray.") - }, - }, - { - desc: "array (error)", - expected: `k=[]`, - f: func(e zapcore.Encoder) { - assert.Error(t, e.AddArray("k", loggable{false, ""}), "Expected an error calling MarshalLogArray.") - }, - }, - { - desc: "reflect enc (success)", - expected: `k\t={"aee":"l=l","bee":123,"cee":0.9999,"dee":[{"key":"p\\ni","val":3.141592653589793},{"key":"tau=","val":6.283185307179586}]}`, - f: func(e zapcore.Encoder) { - type bar struct { - Key string `json:"key"` - Val float64 `json:"val"` - } - - type foo struct { - A string `json:"aee"` - B int `json:"bee"` - C float64 `json:"cee"` - D []bar `json:"dee"` - } - - assert.NoError(t, e.AddReflected("k\t", foo{ - A: "l=l", - B: 123, - C: 0.9999, - D: []bar{ - {"p\ni", 3.141592653589793}, - {"tau=", 6.283185307179586}, - }, - }), "Unexpected error JSON-serializing a map.") - }, - }, - { - desc: "reflect (failure)", - expected: "", - f: func(e zapcore.Encoder) { - assert.Error(t, e.AddReflected("k", noJSON{}), "Unexpected success JSON-serializing a noJSON.") - }, - }, - } - - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - assertOutput(t, tt.expected, tt.f) - }) - } -} - -func TestTskvEncoderOpenNamespace(t *testing.T) { - enc := getRawTskvEncoder() - assert.PanicsWithValue(t, "TSKV encoder does not support namespaces", func() { enc.OpenNamespace("test") }) -} - -func TestTSKVEncoderArrays(t *testing.T) { - tests := []struct { - desc string - expected string // expect f to be called twice - f func(zapcore.ArrayEncoder) - }{ - {"bool", `[true,true]`, func(e zapcore.ArrayEncoder) { e.AppendBool(true) }}, - {"byteString", `[6b,6b]`, func(e zapcore.ArrayEncoder) { e.AppendByteString([]byte("k")) }}, - {"byteString", `[6b5c,6b5c]`, func(e zapcore.ArrayEncoder) { e.AppendByteString([]byte(`k\`)) }}, - {"complex128", `["1+2i","1+2i"]`, func(e zapcore.ArrayEncoder) { e.AppendComplex128(1 + 2i) }}, - {"complex64", `["1+2i","1+2i"]`, func(e zapcore.ArrayEncoder) { e.AppendComplex64(1 + 2i) }}, - {"durations", `[0.000000002,0.000000002]`, func(e zapcore.ArrayEncoder) { e.AppendDuration(2) }}, - {"float64", `[3.14,3.14]`, func(e zapcore.ArrayEncoder) { e.AppendFloat64(3.14) }}, - {"float32", `[3.14,3.14]`, func(e zapcore.ArrayEncoder) { e.AppendFloat32(3.14) }}, - {"int", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendInt(42) }}, - {"int64", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendInt64(42) }}, - {"int32", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendInt32(42) }}, - {"int16", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendInt16(42) }}, - {"int8", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendInt8(42) }}, - {"string", `[k,k]`, func(e zapcore.ArrayEncoder) { e.AppendString("k") }}, - {"string", `[k\\,k\\]`, func(e zapcore.ArrayEncoder) { e.AppendString(`k\`) }}, - {"times", `[1,1]`, func(e zapcore.ArrayEncoder) { e.AppendTime(time.Unix(1, 0)) }}, - {"uint", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUint(42) }}, - {"uint64", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUint64(42) }}, - {"uint32", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUint32(42) }}, - {"uint16", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUint16(42) }}, - {"uint8", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUint8(42) }}, - {"uintptr", `[42,42]`, func(e zapcore.ArrayEncoder) { e.AppendUintptr(42) }}, - { - desc: "arrays (success)", - expected: `[[true],[true]]`, - f: func(arr zapcore.ArrayEncoder) { - assert.NoError(t, arr.AppendArray(zapcore.ArrayMarshalerFunc(func(inner zapcore.ArrayEncoder) error { - inner.AppendBool(true) - return nil - })), "Unexpected error appending an array.") - }, - }, - { - desc: "arrays (error)", - expected: `[[true],[true]]`, - f: func(arr zapcore.ArrayEncoder) { - assert.Error(t, arr.AppendArray(zapcore.ArrayMarshalerFunc(func(inner zapcore.ArrayEncoder) error { - inner.AppendBool(true) - return errors.New("fail") - })), "Expected an error appending an array.") - }, - }, - { - desc: "objects (success)", - expected: `[{loggable=true},{loggable=true}]`, - f: func(arr zapcore.ArrayEncoder) { - assert.NoError(t, arr.AppendObject(loggable{true, ""}), "Unexpected error appending an object.") - }, - }, - { - desc: "objects esc (success)", - expected: `[{loggable\t=true\t},{loggable\t=true\t}]`, - f: func(arr zapcore.ArrayEncoder) { - assert.NoError(t, arr.AppendObject(loggable{true, "\t"}), "Unexpected error appending an object.") - }, - }, - { - desc: "objects (error: fmt.Stringer not implemented)", - expected: `[{},{}]`, - f: func(arr zapcore.ArrayEncoder) { - assert.Error(t, arr.AppendObject(nonloggable{}), "Expected an error appending an object.") - }, - }, - { - desc: "reflect (success)", - expected: `[{"foo":5},{"foo":5}]`, - f: func(arr zapcore.ArrayEncoder) { - assert.NoError( - t, - arr.AppendReflected(map[string]int{"foo": 5}), - "Unexpected an error appending an object with reflection.", - ) - }, - }, - { - desc: "reflect esc (success)", - expected: `[{"foo\\t":5},{"foo\\t":5}]`, - f: func(arr zapcore.ArrayEncoder) { - assert.NoError( - t, - arr.AppendReflected(map[string]int{"foo\t": 5}), - "Unexpected an error appending an object with reflection.", - ) - }, - }, - { - desc: "reflect (error)", - expected: `[]`, - f: func(arr zapcore.ArrayEncoder) { - assert.Error( - t, - arr.AppendReflected(noJSON{}), - "Unexpected an error appending an object with reflection.", - ) - }, - }, - } - - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - f := func(enc zapcore.Encoder) error { - return enc.AddArray("array", zapcore.ArrayMarshalerFunc(func(arr zapcore.ArrayEncoder) error { - tt.f(arr) - tt.f(arr) - return nil - })) - } - assertOutput(t, `array=`+tt.expected, func(enc zapcore.Encoder) { - err := f(enc) - assert.NoError(t, err, "Unexpected error adding array to JSON encoder.") - }) - }) - } -} - -func TestTSKVEncodeEntry(t *testing.T) { - entryTime := time.Date(2019, 7, 13, 15, 33, 42, 99, time.UTC) - - tests := []struct { - desc string - expected string - cnf zapcore.EncoderConfig - ent zapcore.Entry - fields []zapcore.Field - }{ - { - desc: "entry without escape", - expected: `tskv T=1563032022 L=info M=text here -`, - cnf: getCommonTestConfig(), - ent: zapcore.Entry{ - Time: entryTime, - Message: "text here", - }, - fields: []zapcore.Field{}, - }, - { - desc: "all fields entry without escape", - expected: `tskv T=1563032022 L=debug N=bob C=foo.go:42 M=text here S=fake-stack -`, - cnf: getCommonTestConfig(), - ent: zapcore.Entry{ - Level: zapcore.DebugLevel, - Time: entryTime, - LoggerName: "bob", - Message: "text here", - Caller: zapcore.EntryCaller{Defined: true, File: "foo.go", Line: 42}, - Stack: "fake-stack", - }, - fields: []zapcore.Field{}, - }, - { - desc: "entry with escaped field names", - expected: `tskv T\r=1563032022 L\n=debug N\0=bob C\==foo.go:42 M\t=text here S\\=fake-stack -`, - cnf: getSpecCharsTestConfig(), - ent: zapcore.Entry{ - Level: zapcore.DebugLevel, - Time: entryTime, - LoggerName: "bob", - Message: "text here", - Caller: zapcore.EntryCaller{Defined: true, File: "foo.go", Line: 42}, - Stack: "fake-stack", - }, - fields: []zapcore.Field{}, - }, - { - desc: "entry message escape", - expected: `tskv T=1563032022 L=info M=t\\ex=t\0he\r\tre\n -`, - cnf: getCommonTestConfig(), - ent: zapcore.Entry{ - Time: entryTime, - Message: "t\\ex=t\x00he\r\tre\n", - }, - fields: []zapcore.Field{}, - }, - { - desc: "entry multi-line stack escape", - expected: `tskv T=1563032022 L=info M= S=fake-st\rack\n\tlevel2\n\tlevel1 -`, - cnf: getCommonTestConfig(), - ent: zapcore.Entry{ - Time: entryTime, - Stack: "fake-st\rack\n\tlevel2\n\tlevel1", - }, - fields: []zapcore.Field{}, - }, - { - desc: "entry multi-line caller escape", - expected: `tskv T=1563032022 L=info C=fo\to.go:42 M= -`, - cnf: getCommonTestConfig(), - ent: zapcore.Entry{ - Time: entryTime, - Caller: zapcore.EntryCaller{Defined: true, File: "fo\to.go", Line: 42}, - }, - fields: []zapcore.Field{}, - }, - { - desc: "entry multi-line logger escape", - expected: `tskv T=1563032022 L=info N=b\0b M= -`, - cnf: getCommonTestConfig(), - ent: zapcore.Entry{ - Time: entryTime, - LoggerName: "b\x00b", - }, - fields: []zapcore.Field{}, - }, - { - desc: "entry with additional zap fields", - expected: `tskv T=1563032022 L=info M= so=passes answer=42 common_pie=3.14 ` + - `reflect={"loggable":"yes"} bytes_array=0001020309 bool=true complex="0+1i" -`, - cnf: getCommonTestConfig(), - ent: zapcore.Entry{ - Time: entryTime, - }, - fields: []zapcore.Field{ - zap.String("so", "passes"), - zap.Int("answer", 42), - zap.Float64("common_pie", 3.14), - zap.Reflect("reflect", map[string]string{"loggable": "yes"}), - zap.Binary("bytes_array", []byte{0, 1, 2, 3, '\t'}), - zap.Bool("bool", true), - zap.Complex128("complex", 1i)}, - }, - } - - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - enc, err := NewTSKVEncoder(tt.cnf) - if err != nil { - panic(err) - } - - buf, err := enc.EncodeEntry(tt.ent, tt.fields) - if assert.NoError(t, err, "Unexpected TSKV encoding error.") { - assert.Equal(t, tt.expected, buf.String(), "Incorrect encoded TSKV entry.") - } - buf.Free() - }) - } -} - -func TestTskvEncoderLoggerWithMethod(t *testing.T) { - entryTime := time.Date(2019, 7, 13, 15, 33, 42, 99, time.UTC) - - enc := getRawTskvEncoder() - enc.AddString("Permanent", "message") - enc.Clone() - tt := struct { - desc string - expected string - ent zapcore.Entry - }{ - desc: "entry without escape", - expected: `tskv T=1563032022 L=info Permanent=message M=text here -`, - ent: zapcore.Entry{ - Time: entryTime, - Message: "text here", - }, - } - - for i := 0; i < 3; i++ { - t.Run(tt.desc, func(t *testing.T) { - buf, err := enc.EncodeEntry(tt.ent, []zapcore.Field{}) - if assert.NoError(t, err, "Unexpected TSKV encoding error.") { - assert.Equal(t, tt.expected, buf.String(), "Incorrect encoded TSKV entry.") - } - }) - } -} diff --git a/library/go/core/log/zap/logrotate/error.go b/library/go/core/log/zap/logrotate/error.go deleted file mode 100644 index f59b3225..00000000 --- a/library/go/core/log/zap/logrotate/error.go +++ /dev/null @@ -1,5 +0,0 @@ -package logrotate - -import "errors" - -var ErrNotSupported = errors.New("logrotate sink is not supported on your platform") diff --git a/library/go/core/log/zap/logrotate/example_sink_test.go b/library/go/core/log/zap/logrotate/example_sink_test.go deleted file mode 100644 index e584a7f6..00000000 --- a/library/go/core/log/zap/logrotate/example_sink_test.go +++ /dev/null @@ -1,54 +0,0 @@ -//go:build linux || darwin -// +build linux darwin - -package logrotate_test - -import ( - "net/url" - "path/filepath" - "syscall" - - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "github.com/ydb-platform/fq-connector-go/library/go/core/log/zap" - "github.com/ydb-platform/fq-connector-go/library/go/core/log/zap/logrotate" - uberzap "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -func Example_simpleUsage() { - // Basic usage, when you don't need any custom preferences is quite easy. - // register our logrotate sink and force it to reopen files on sighup(remember to check for errors) - _ = logrotate.RegisterLogrotateSink(syscall.SIGHUP) - // create zap logger as usual, using `logrotate://` instead of omitting it or using `file://` - cfg := zap.JSONConfig(log.DebugLevel) - logPath, _ := filepath.Abs("./example.log") - cfg.OutputPaths = []string{"logrotate://" + logPath} - logger, _ := zap.New(cfg) - // That's all, when your process receives SIGHUP file will be reopened - logger.Debug("this log should be reopened by SIGHUP") -} - -func Example_namedUsage() { - // Note: each scheme can be registered only once and can not be unregistered - // If you want to provide custom unused scheme name(remember to check for errors): - _ = logrotate.RegisterNamedLogrotateSink("rotate-usr1", syscall.SIGUSR1) - // Now we create logger using that cheme - cfg := zap.JSONConfig(log.DebugLevel) - logPath, _ := filepath.Abs("./example.log") - cfg.OutputPaths = []string{"rotate-usr1://" + logPath} - logger, _ := zap.New(cfg) - // Now file will be reopened by SIGUSR1 - logger.Debug("this log should be reopened by SIGHUP") -} - -func Example_standaloneUsage() { - // If you don't want to register scheme, or use custom logging core you can do this(remember to check for errors): - u, _ := url.ParseRequestURI("/tmp/example.log") - sink, _ := logrotate.NewLogrotateSink(u, syscall.SIGHUP) - - encoder := zapcore.NewConsoleEncoder(zapcore.EncoderConfig{MessageKey: "msg"}) - core := zapcore.NewCore(encoder, sink, uberzap.NewAtomicLevel()) - logger := uberzap.New(core) - // Now file will be reopened by SIGHUP - logger.Debug("this log should be reopened by SIGHUP") -} diff --git a/library/go/core/log/zap/logrotate/sink.go b/library/go/core/log/zap/logrotate/sink.go deleted file mode 100644 index 80095908..00000000 --- a/library/go/core/log/zap/logrotate/sink.go +++ /dev/null @@ -1,121 +0,0 @@ -//go:build darwin || freebsd || linux -// +build darwin freebsd linux - -package logrotate - -import ( - "fmt" - "net/url" - "os" - "os/signal" - "sync/atomic" - "unsafe" - - "github.com/ydb-platform/fq-connector-go/library/go/core/xerrors" - "go.uber.org/zap" -) - -const defaultSchemeName = "logrotate" - -// Register logrotate sink in zap sink registry. -// This sink internally is like file sink, but listens to provided logrotate signal -// and reopens file when that signal is delivered -// This can be called only once. Any future calls will result in an error -func RegisterLogrotateSink(sig ...os.Signal) error { - return RegisterNamedLogrotateSink(defaultSchemeName, sig...) -} - -// Same as RegisterLogrotateSink, but use provided schemeName instead of default `logrotate` -// Can be useful in special cases for registering different types of sinks for different signal -func RegisterNamedLogrotateSink(schemeName string, sig ...os.Signal) error { - factory := func(url *url.URL) (sink zap.Sink, e error) { - return NewLogrotateSink(url, sig...) - } - return zap.RegisterSink(schemeName, factory) -} - -// sink itself, use RegisterLogrotateSink to register it in zap machinery -type sink struct { - path string - notifier chan os.Signal - file unsafe.Pointer -} - -// Factory for logrotate sink, which accepts os.Signals to listen to for reloading -// Generally if you don't build your own core it is used by zap machinery. -// See RegisterLogrotateSink. -func NewLogrotateSink(u *url.URL, sig ...os.Signal) (zap.Sink, error) { - notifier := make(chan os.Signal, 1) - signal.Notify(notifier, sig...) - - if u.User != nil { - return nil, fmt.Errorf("user and password not allowed with logrotate file URLs: got %v", u) - } - if u.Fragment != "" { - return nil, fmt.Errorf("fragments not allowed with logrotate file URLs: got %v", u) - } - // Error messages are better if we check hostname and port separately. - if u.Port() != "" { - return nil, fmt.Errorf("ports not allowed with logrotate file URLs: got %v", u) - } - if hn := u.Hostname(); hn != "" && hn != "localhost" { - return nil, fmt.Errorf("logrotate file URLs must leave host empty or use localhost: got %v", u) - } - - sink := &sink{ - path: u.Path, - notifier: notifier, - } - if err := sink.reopen(); err != nil { - return nil, err - } - go sink.listenToSignal() - return sink, nil -} - -// wait for signal delivery or chanel close -func (m *sink) listenToSignal() { - for { - _, ok := <-m.notifier - if !ok { - return - } - if err := m.reopen(); err != nil { - // Last chance to signalize about an error - _, _ = fmt.Fprintf(os.Stderr, "%s", err) - } - } -} - -func (m *sink) reopen() error { - file, err := os.OpenFile(m.path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644) - if err != nil { - return xerrors.Errorf("failed to open log file on %s: %w", m.path, err) - } - old := (*os.File)(m.file) - atomic.StorePointer(&m.file, unsafe.Pointer(file)) - if old != nil { - if err := old.Close(); err != nil { - return xerrors.Errorf("failed to close old file: %w", err) - } - } - return nil -} - -func (m *sink) getFile() *os.File { - return (*os.File)(atomic.LoadPointer(&m.file)) -} - -func (m *sink) Close() error { - signal.Stop(m.notifier) - close(m.notifier) - return m.getFile().Close() -} - -func (m *sink) Write(p []byte) (n int, err error) { - return m.getFile().Write(p) -} - -func (m *sink) Sync() error { - return m.getFile().Sync() -} diff --git a/library/go/core/log/zap/logrotate/sink_test.go b/library/go/core/log/zap/logrotate/sink_test.go deleted file mode 100644 index 1461e9e0..00000000 --- a/library/go/core/log/zap/logrotate/sink_test.go +++ /dev/null @@ -1,86 +0,0 @@ -//go:build linux || darwin -// +build linux darwin - -package logrotate - -import ( - "io" - "os" - "path/filepath" - "strings" - "syscall" - "testing" - "time" - - "github.com/stretchr/testify/require" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "github.com/ydb-platform/fq-connector-go/library/go/core/log/zap" -) - -func TestLogrotateSink(t *testing.T) { - testLogFilename := "test.log" - testDir := "testLogrotate" - - // use test dir in default temp files location - tempDir, err := os.MkdirTemp("", testDir) - require.NoError(t, err, "failed to create temporary directory %s", testDir) - - testLogPath := filepath.Join(tempDir, testLogFilename) - - defer func() { - _ = os.RemoveAll(tempDir) - }() // clean up - - err = RegisterLogrotateSink(syscall.SIGUSR1) - require.NoError(t, err, "failed to register sink") - - // Double registration is not allowed - err = RegisterLogrotateSink(syscall.SIGUSR1) - require.Error(t, err) - - cfg := zap.JSONConfig(log.DebugLevel) - cfg.OutputPaths = []string{"logrotate://" + testLogPath} - logger, err := zap.New(cfg) - require.NoError(t, err, "failed to create logger") - - testLogFile, err := os.OpenFile(testLogPath, os.O_RDONLY, 0) - require.NoError(t, err, "expected logger to create file: %v", err) - defer func() { - _ = testLogFile.Close() - }() - - // test write to file - logger.Debug("test") - logger.Debug("test") - - err = os.Rename(testLogPath, testLogPath+".rotated") - require.NoError(t, err, "failed to rename file") - - err = syscall.Kill(syscall.Getpid(), syscall.SIGUSR1) - require.NoError(t, err, "failed to send signal to self, %v", err) - - // There is an essential race that we can not control of delivering signal, - // so we just wait enough here - time.Sleep(time.Second) - - logger.Debug("test") - logger.Debug("test") - logger.Debug("test") - - // Reopen file to sync content - err = syscall.Kill(syscall.Getpid(), syscall.SIGUSR1) - require.NoError(t, err, "failed to send signal to self, %v", err) - time.Sleep(time.Second) - - requireLineCount(t, testLogPath, 3) - requireLineCount(t, testLogPath+".rotated", 2) -} - -func requireLineCount(t *testing.T, path string, lines int) { - file, err := os.OpenFile(path, os.O_RDONLY, 0) - require.NoError(t, err, "failed to open log file for reading") - defer func() { _ = file.Close() }() - dataRead, err := io.ReadAll(file) - require.NoError(t, err, "failed to read log file") - require.Equal(t, lines, strings.Count(string(dataRead), "\n")) -} diff --git a/library/go/core/log/zap/qloud.go b/library/go/core/log/zap/qloud.go deleted file mode 100644 index 09d686e3..00000000 --- a/library/go/core/log/zap/qloud.go +++ /dev/null @@ -1,49 +0,0 @@ -package zap - -import ( - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -// NewQloudLogger constructs fully-fledged Qloud compatible logger -// based on predefined config. See https://wiki.yandex-team.ru/qloud/doc/logs -// for more information -func NewQloudLogger(level log.Level, opts ...zap.Option) (*Logger, error) { - cfg := zap.Config{ - Level: zap.NewAtomicLevelAt(ZapifyLevel(level)), - Encoding: "json", - OutputPaths: []string{"stdout"}, - ErrorOutputPaths: []string{"stderr"}, - EncoderConfig: zapcore.EncoderConfig{ - MessageKey: "msg", - LevelKey: "level", - StacktraceKey: "stackTrace", - TimeKey: "", - CallerKey: "", - EncodeLevel: zapcore.LowercaseLevelEncoder, - EncodeTime: zapcore.ISO8601TimeEncoder, - EncodeDuration: zapcore.StringDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - }, - } - - zl, err := cfg.Build(opts...) - if err != nil { - return nil, err - } - - return &Logger{ - L: addQloudContext(zl).(*zap.Logger), - }, nil -} - -func addQloudContext(i interface{}) interface{} { - switch c := i.(type) { - case *zap.Logger: - return c.With(zap.Namespace("@fields")) - case zapcore.Core: - return c.With([]zapcore.Field{zap.Namespace("@fields")}) - } - return i -} diff --git a/library/go/core/log/zap/zap.go b/library/go/core/log/zap/zap.go deleted file mode 100644 index be5965d1..00000000 --- a/library/go/core/log/zap/zap.go +++ /dev/null @@ -1,252 +0,0 @@ -package zap - -import ( - "fmt" - - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "github.com/ydb-platform/fq-connector-go/library/go/core/log/zap/encoders" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -const ( - // callerSkip is number of stack frames to skip when logging caller - callerSkip = 1 -) - -func init() { - if err := zap.RegisterEncoder(encoders.EncoderNameKV, encoders.NewKVEncoder); err != nil { - panic(err) - } - if err := zap.RegisterEncoder(encoders.EncoderNameCli, encoders.NewCliEncoder); err != nil { - panic(err) - } - if err := zap.RegisterEncoder(encoders.EncoderNameTSKV, encoders.NewTSKVEncoder); err != nil { - panic(err) - } -} - -// Logger implements log.Logger interface -type Logger struct { - L *zap.Logger -} - -var _ log.Logger = &Logger{} -var _ log.Structured = &Logger{} -var _ log.Fmt = &Logger{} -var _ log.LoggerWith = &Logger{} -var _ log.LoggerAddCallerSkip = &Logger{} - -// New constructs zap-based logger from provided config -func New(cfg zap.Config) (*Logger, error) { - zl, err := cfg.Build(zap.AddCallerSkip(callerSkip)) - if err != nil { - return nil, err - } - - return &Logger{ - L: zl, - }, nil -} - -// NewWithCore constructs zap-based logger from provided core -func NewWithCore(core zapcore.Core, options ...zap.Option) *Logger { - options = append(options, zap.AddCallerSkip(callerSkip)) - return &Logger{L: zap.New(core, options...)} -} - -// Must constructs zap-based logger from provided config and panics on error -func Must(cfg zap.Config) *Logger { - l, err := New(cfg) - if err != nil { - panic(fmt.Sprintf("failed to construct zap logger: %v", err)) - } - return l -} - -// JSONConfig returns zap config for structured logging (zap's json encoder) -func JSONConfig(level log.Level) zap.Config { - return StandardConfig("json", level) -} - -// ConsoleConfig returns zap config for logging to console (zap's console encoder) -func ConsoleConfig(level log.Level) zap.Config { - return StandardConfig("console", level) -} - -// CLIConfig returns zap config for cli logging (custom cli encoder) -func CLIConfig(level log.Level) zap.Config { - return StandardConfig("cli", level) -} - -// KVConfig returns zap config for logging to kv (custom kv encoder) -func KVConfig(level log.Level) zap.Config { - return StandardConfig("kv", level) -} - -// TSKVConfig returns zap config for logging to tskv (custom tskv encoder) -func TSKVConfig(level log.Level) zap.Config { - return zap.Config{ - Level: zap.NewAtomicLevelAt(ZapifyLevel(level)), - Encoding: "tskv", - OutputPaths: []string{"stdout"}, - ErrorOutputPaths: []string{"stderr"}, - EncoderConfig: zapcore.EncoderConfig{ - MessageKey: "message", - LevelKey: "levelname", - TimeKey: "unixtime", - CallerKey: "caller", - NameKey: "name", - EncodeLevel: zapcore.CapitalLevelEncoder, - EncodeTime: zapcore.EpochTimeEncoder, - EncodeDuration: zapcore.StringDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - }, - } -} - -// StandardConfig returns default zap config with specified encoding and level -func StandardConfig(encoding string, level log.Level) zap.Config { - return zap.Config{ - Level: zap.NewAtomicLevelAt(ZapifyLevel(level)), - Encoding: encoding, - OutputPaths: []string{"stdout"}, - ErrorOutputPaths: []string{"stderr"}, - EncoderConfig: zapcore.EncoderConfig{ - MessageKey: "msg", - LevelKey: "level", - TimeKey: "ts", - CallerKey: "caller", - NameKey: "name", - EncodeLevel: zapcore.CapitalLevelEncoder, - EncodeTime: zapcore.ISO8601TimeEncoder, - EncodeDuration: zapcore.StringDurationEncoder, - EncodeCaller: zapcore.ShortCallerEncoder, - }, - } -} - -// Logger returns general logger -func (l *Logger) Logger() log.Logger { - return l -} - -// Fmt returns fmt logger -func (l *Logger) Fmt() log.Fmt { - return l -} - -// Structured returns structured logger -func (l *Logger) Structured() log.Structured { - return l -} - -// With returns logger that always adds provided key/value to every log entry -func (l *Logger) With(fields ...log.Field) log.Logger { - return &Logger{ - L: l.L.With(zapifyFields(fields...)...), - } -} - -func (l *Logger) AddCallerSkip(skip int) log.Logger { - return &Logger{ - L: l.L.WithOptions(zap.AddCallerSkip(skip)), - } -} - -// Trace logs at Trace log level using fields -func (l *Logger) Trace(msg string, fields ...log.Field) { - if ce := l.L.Check(zap.DebugLevel, msg); ce != nil { - ce.Write(zapifyFields(fields...)...) - } -} - -// Tracef logs at Trace log level using fmt formatter -func (l *Logger) Tracef(msg string, args ...interface{}) { - if ce := l.L.Check(zap.DebugLevel, ""); ce != nil { - ce.Message = fmt.Sprintf(msg, args...) - ce.Write() - } -} - -// Debug logs at Debug log level using fields -func (l *Logger) Debug(msg string, fields ...log.Field) { - if ce := l.L.Check(zap.DebugLevel, msg); ce != nil { - ce.Write(zapifyFields(fields...)...) - } -} - -// Debugf logs at Debug log level using fmt formatter -func (l *Logger) Debugf(msg string, args ...interface{}) { - if ce := l.L.Check(zap.DebugLevel, ""); ce != nil { - ce.Message = fmt.Sprintf(msg, args...) - ce.Write() - } -} - -// Info logs at Info log level using fields -func (l *Logger) Info(msg string, fields ...log.Field) { - if ce := l.L.Check(zap.InfoLevel, msg); ce != nil { - ce.Write(zapifyFields(fields...)...) - } -} - -// Infof logs at Info log level using fmt formatter -func (l *Logger) Infof(msg string, args ...interface{}) { - if ce := l.L.Check(zap.InfoLevel, ""); ce != nil { - ce.Message = fmt.Sprintf(msg, args...) - ce.Write() - } -} - -// Warn logs at Warn log level using fields -func (l *Logger) Warn(msg string, fields ...log.Field) { - if ce := l.L.Check(zap.WarnLevel, msg); ce != nil { - ce.Write(zapifyFields(fields...)...) - } -} - -// Warnf logs at Warn log level using fmt formatter -func (l *Logger) Warnf(msg string, args ...interface{}) { - if ce := l.L.Check(zap.WarnLevel, ""); ce != nil { - ce.Message = fmt.Sprintf(msg, args...) - ce.Write() - } -} - -// Error logs at Error log level using fields -func (l *Logger) Error(msg string, fields ...log.Field) { - if ce := l.L.Check(zap.ErrorLevel, msg); ce != nil { - ce.Write(zapifyFields(fields...)...) - } -} - -// Errorf logs at Error log level using fmt formatter -func (l *Logger) Errorf(msg string, args ...interface{}) { - if ce := l.L.Check(zap.ErrorLevel, ""); ce != nil { - ce.Message = fmt.Sprintf(msg, args...) - ce.Write() - } -} - -// Fatal logs at Fatal log level using fields -func (l *Logger) Fatal(msg string, fields ...log.Field) { - if ce := l.L.Check(zap.FatalLevel, msg); ce != nil { - ce.Write(zapifyFields(fields...)...) - } -} - -// Fatalf logs at Fatal log level using fmt formatter -func (l *Logger) Fatalf(msg string, args ...interface{}) { - if ce := l.L.Check(zap.FatalLevel, ""); ce != nil { - ce.Message = fmt.Sprintf(msg, args...) - ce.Write() - } -} - -// WithName adds name to logger -func (l *Logger) WithName(name string) log.Logger { - return &Logger{ - L: l.L.Named(name), - } -} diff --git a/library/go/core/log/zap/zap_test.go b/library/go/core/log/zap/zap_test.go deleted file mode 100644 index dbe803df..00000000 --- a/library/go/core/log/zap/zap_test.go +++ /dev/null @@ -1,113 +0,0 @@ -package zap - -import ( - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/assert" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" - "go.uber.org/zap/zaptest/observer" -) - -func TestNewQloudLogger(t *testing.T) { - logger, err := NewQloudLogger(log.DebugLevel) - assert.NoError(t, err) - - core, logs := observer.New(zap.DebugLevel) - - logger.L = logger.L.WithOptions(zap.WrapCore(func(c zapcore.Core) zapcore.Core { - return addQloudContext(core).(zapcore.Core) - })) - - expectedMessage := "test message" - - logger.Info(expectedMessage, log.String("package", "zap")) - assert.Equal(t, 1, logs.Len()) - - loggedEntry := logs.AllUntimed()[0] - assert.Equal(t, zap.InfoLevel, loggedEntry.Level) - assert.Equal(t, expectedMessage, loggedEntry.Message) - assert.Equal(t, - map[string]interface{}{ - "@fields": map[string]interface{}{ - "package": "zap", - }, - }, - loggedEntry.ContextMap(), - ) -} - -func TestLogger_FormattedMethods(t *testing.T) { - testCases := []struct { - lvl log.Level - expectLogged []zapcore.Entry - }{ - {log.TraceLevel, []zapcore.Entry{ - {Level: zap.DebugLevel, Message: "test at trace"}, - {Level: zap.DebugLevel, Message: "test at debug"}, - {Level: zap.InfoLevel, Message: "test at info"}, - {Level: zap.WarnLevel, Message: "test at warn"}, - {Level: zap.ErrorLevel, Message: "test at error"}, - }}, - {log.DebugLevel, []zapcore.Entry{ - {Level: zap.DebugLevel, Message: "test at trace"}, - {Level: zap.DebugLevel, Message: "test at debug"}, - {Level: zap.InfoLevel, Message: "test at info"}, - {Level: zap.WarnLevel, Message: "test at warn"}, - {Level: zap.ErrorLevel, Message: "test at error"}, - }}, - {log.InfoLevel, []zapcore.Entry{ - {Level: zap.InfoLevel, Message: "test at info"}, - {Level: zap.WarnLevel, Message: "test at warn"}, - {Level: zap.ErrorLevel, Message: "test at error"}, - }}, - {log.WarnLevel, []zapcore.Entry{ - {Level: zap.WarnLevel, Message: "test at warn"}, - {Level: zap.ErrorLevel, Message: "test at error"}, - }}, - {log.ErrorLevel, []zapcore.Entry{ - {Level: zap.ErrorLevel, Message: "test at error"}, - }}, - } - - for _, tc := range testCases { - t.Run(tc.lvl.String(), func(t *testing.T) { - logger, err := New(ConsoleConfig(tc.lvl)) - assert.NoError(t, err) - - core, logs := observer.New(ZapifyLevel(tc.lvl)) - - logger.L = logger.L.WithOptions(zap.WrapCore(func(_ zapcore.Core) zapcore.Core { - return core - })) - - for _, lvl := range log.Levels() { - switch lvl { - case log.TraceLevel: - logger.Tracef("test at %s", lvl.String()) - case log.DebugLevel: - logger.Debugf("test at %s", lvl.String()) - case log.InfoLevel: - logger.Infof("test at %s", lvl.String()) - case log.WarnLevel: - logger.Warnf("test at %s", lvl.String()) - case log.ErrorLevel: - logger.Errorf("test at %s", lvl.String()) - case log.FatalLevel: - // skipping fatal - } - } - - loggedEntries := logs.AllUntimed() - - assert.Equal(t, len(tc.expectLogged), logs.Len(), cmp.Diff(tc.expectLogged, loggedEntries)) - - for i, le := range loggedEntries { - assert.Equal(t, tc.expectLogged[i].Level, le.Level) - assert.Equal(t, tc.expectLogged[i].Message, le.Message) - } - }) - } -} diff --git a/library/go/core/log/zap/zapify.go b/library/go/core/log/zap/zapify.go deleted file mode 100644 index 9865cdae..00000000 --- a/library/go/core/log/zap/zapify.go +++ /dev/null @@ -1,98 +0,0 @@ -package zap - -import ( - "context" - "fmt" - - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" -) - -// ZapifyLevel turns interface log level to zap log level -func ZapifyLevel(level log.Level) zapcore.Level { - switch level { - case log.TraceLevel: - return zapcore.DebugLevel - case log.DebugLevel: - return zapcore.DebugLevel - case log.InfoLevel: - return zapcore.InfoLevel - case log.WarnLevel: - return zapcore.WarnLevel - case log.ErrorLevel: - return zapcore.ErrorLevel - case log.FatalLevel: - return zapcore.FatalLevel - default: - // For when new log level is not added to this func (most likely never). - panic(fmt.Sprintf("unknown log level: %d", level)) - } -} - -// UnzapifyLevel turns zap log level to interface log level. -func UnzapifyLevel(level zapcore.Level) log.Level { - switch level { - case zapcore.DebugLevel: - return log.DebugLevel - case zapcore.InfoLevel: - return log.InfoLevel - case zapcore.WarnLevel: - return log.WarnLevel - case zapcore.ErrorLevel: - return log.ErrorLevel - case zapcore.FatalLevel, zapcore.DPanicLevel, zapcore.PanicLevel: - return log.FatalLevel - default: - // For when new log level is not added to this func (most likely never). - panic(fmt.Sprintf("unknown log level: %d", level)) - } -} - -// nolint: gocyclo -func zapifyField(field log.Field) zap.Field { - switch field.Type() { - case log.FieldTypeNil: - return zap.Reflect(field.Key(), nil) - case log.FieldTypeString: - return zap.String(field.Key(), field.String()) - case log.FieldTypeBinary: - return zap.Binary(field.Key(), field.Binary()) - case log.FieldTypeBoolean: - return zap.Bool(field.Key(), field.Bool()) - case log.FieldTypeSigned: - return zap.Int64(field.Key(), field.Signed()) - case log.FieldTypeUnsigned: - return zap.Uint64(field.Key(), field.Unsigned()) - case log.FieldTypeFloat: - return zap.Float64(field.Key(), field.Float()) - case log.FieldTypeTime: - return zap.Time(field.Key(), field.Time()) - case log.FieldTypeDuration: - return zap.Duration(field.Key(), field.Duration()) - case log.FieldTypeError: - return zap.NamedError(field.Key(), field.Error()) - case log.FieldTypeArray: - return zap.Any(field.Key(), field.Interface()) - case log.FieldTypeAny: - return zap.Any(field.Key(), field.Interface()) - case log.FieldTypeReflect: - return zap.Reflect(field.Key(), field.Interface()) - case log.FieldTypeByteString: - return zap.ByteString(field.Key(), field.Binary()) - case log.FieldTypeContext: - return Context(field.Interface().(context.Context)) - default: - // For when new field type is not added to this func - panic(fmt.Sprintf("unknown field type: %d", field.Type())) - } -} - -func zapifyFields(fields ...log.Field) []zapcore.Field { - zapFields := make([]zapcore.Field, 0, len(fields)) - for _, field := range fields { - zapFields = append(zapFields, zapifyField(field)) - } - - return zapFields -} diff --git a/library/go/core/log/zap/zapify_test.go b/library/go/core/log/zap/zapify_test.go deleted file mode 100644 index ff409f05..00000000 --- a/library/go/core/log/zap/zapify_test.go +++ /dev/null @@ -1,60 +0,0 @@ -package zap - -import ( - "fmt" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/ydb-platform/fq-connector-go/library/go/core/log" - "go.uber.org/zap/zapcore" -) - -// Simple test, that all type of fields are correctly zapified. -// Maybe we also need some test that checks resulting zap.Field type also. -func TestZapifyField(t *testing.T) { - fileds := []log.Field{ - log.Nil("test"), - log.String("test", "test"), - log.Binary("test", []byte("test")), - log.Bool("test", true), - log.Int("test", -42), - log.UInt("test", 42), - log.Float64("test", 0.42), - log.Time("test", time.Now()), - log.Duration("test", time.Second), - log.Error(fmt.Errorf("test")), - log.Array("test", []uint32{42}), - log.Any("test", struct{ ID uint32 }{ID: 42}), - log.Reflect("test", struct{ ID uint32 }{ID: 42}), - } - for _, field := range fileds { - assert.NotPanics(t, func() { - zapifyField(field) - }) - } -} - -func TestZapifyAny(t *testing.T) { - f := zapifyField(log.Any("test", struct{ ID uint32 }{ID: 42})) - assert.Equal(t, zapcore.ReflectType, f.Type) -} - -func TestZapifyReflect(t *testing.T) { - f := zapifyField(log.Any("test", struct{ ID uint32 }{ID: 42})) - assert.Equal(t, zapcore.ReflectType, f.Type) -} - -type stringer struct{} - -func (*stringer) String() string { - return "hello" -} - -func TestZapifyStringer(t *testing.T) { - f0 := zapifyField(log.Any("test", &stringer{})) - assert.Equal(t, zapcore.StringerType, f0.Type) - - f1 := zapifyField(log.Reflect("test", &stringer{})) - assert.Equal(t, zapcore.ReflectType, f1.Type) -}