From 310ee6e12be4c9aeb12c319a0262d80ea24201d0 Mon Sep 17 00:00:00 2001 From: Nurlan Moldomurov Date: Wed, 11 Dec 2024 13:07:21 +0300 Subject: [PATCH] PMM-13602 Improve performance (#987) * PMM-13602 Improve performance * PMM-13602 Improve performance for callstats * PMM-13602 enable indexDetails. * PMM-13602 fix linters. * PMM-13602 revert diagnostic data changes. * PMM-13602 fix linters. * Update .golangci.yml Co-authored-by: Alex Demidoff --------- Co-authored-by: Alex Demidoff --- .golangci.yml | 3 +- exporter/collstats_collector.go | 8 +- exporter/collstats_collector_test.go | 2 +- exporter/exporter.go | 3 +- exporter/metrics.go | 106 ++- exporter/metrics_test.go | 2 +- exporter/top_collector.go | 2 +- exporter/v1_compatibility.go | 998 +++++++++++++-------------- 8 files changed, 559 insertions(+), 565 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index a45a5dbc5..1c55c4bb6 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -32,8 +32,9 @@ linters: - unused - testpackage - wsl - - exhaustivestruct + - exhaustruct - varnamelen + - gochecknoglobals # we need globals for better performance, we use them for mapping, and we don't mutate them, so it's safe. - maligned #deprecated - scopelint #deprecated - golint #deprecated diff --git a/exporter/collstats_collector.go b/exporter/collstats_collector.go index 882260765..dd51e3440 100644 --- a/exporter/collstats_collector.go +++ b/exporter/collstats_collector.go @@ -37,12 +37,12 @@ type collstatsCollector struct { } // newCollectionStatsCollector creates a collector for statistics about collections. -func newCollectionStatsCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, compatible, discovery bool, topology labelsGetter, collections []string) *collstatsCollector { +func newCollectionStatsCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, discovery bool, topology labelsGetter, collections []string) *collstatsCollector { return &collstatsCollector{ ctx: ctx, base: newBaseCollector(client, logger.WithFields(logrus.Fields{"collector": "collstats"})), - compatibleMode: compatible, + compatibleMode: false, // there are no compatible metrics for this collector. discoveringMode: discovery, topologyInfo: topology, @@ -108,7 +108,9 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) { }, } - cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, mongo.Pipeline{aggregation}) + pipeline := mongo.Pipeline{aggregation} + + cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, pipeline) if err != nil { logger.Errorf("cannot get $collstats cursor for collection %s.%s: %s", database, collection, err) diff --git a/exporter/collstats_collector_test.go b/exporter/collstats_collector_test.go index 8da41c9e8..f1dcaddc7 100644 --- a/exporter/collstats_collector_test.go +++ b/exporter/collstats_collector_test.go @@ -54,7 +54,7 @@ func TestCollStatsCollector(t *testing.T) { collection := []string{"testdb.testcol_00", "testdb.testcol_01", "testdb.testcol_02"} logger := logrus.New() - c := newCollectionStatsCollector(ctx, client, logger, false, false, ti, collection) + c := newCollectionStatsCollector(ctx, client, logger, false, ti, collection) // The last \n at the end of this string is important expected := strings.NewReader(` diff --git a/exporter/exporter.go b/exporter/exporter.go index dfed85093..b7c7bce2a 100644 --- a/exporter/exporter.go +++ b/exporter/exporter.go @@ -153,6 +153,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol e.getTotalCollectionsCount() <= e.opts.CollStatsLimit { limitsOk = true } + limitsOk = true if e.opts.CollectAll { if len(e.opts.CollStatsNamespaces) == 0 { @@ -191,7 +192,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol // If we manually set the collection names we want or auto discovery is set. if (len(e.opts.CollStatsNamespaces) > 0 || e.opts.DiscoveringMode) && e.opts.EnableCollStats && limitsOk && requestOpts.EnableCollStats { cc := newCollectionStatsCollector(ctx, client, e.opts.Logger, - e.opts.CompatibleMode, e.opts.DiscoveringMode, + e.opts.DiscoveringMode, topologyInfo, e.opts.CollStatsNamespaces) registry.MustRegister(cc) } diff --git a/exporter/metrics.go b/exporter/metrics.go index 74e1ea6d1..cd3d27246 100644 --- a/exporter/metrics.go +++ b/exporter/metrics.go @@ -335,7 +335,7 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo metrics := []*rawMetric{rm} - if renamedMetrics := metricRenameAndLabel(rm, specialConversions()); renamedMetrics != nil { + if renamedMetrics := metricRenameAndLabel(rm, specialConversions); renamedMetrics != nil { metrics = renamedMetrics } @@ -435,62 +435,60 @@ func metricRenameAndLabel(rm *rawMetric, convs []conversion) []*rawMetric { // specialConversions returns a list of special conversions we want to implement. // See: https://jira.percona.com/browse/PMM-6506 -func specialConversions() []conversion { - return []conversion{ - { - oldName: "mongodb_ss_opLatencies_ops", - prefix: "mongodb_ss_opLatencies", - suffixLabel: "op_type", - suffixMapping: map[string]string{ - "commands_ops": "commands", - "reads_ops": "reads", - "transactions_ops": "transactions", - "writes_ops": "writes", - }, +var specialConversions = []conversion{ //nolint:gochecknoglobals + { + oldName: "mongodb_ss_opLatencies_ops", + prefix: "mongodb_ss_opLatencies", + suffixLabel: "op_type", + suffixMapping: map[string]string{ + "commands_ops": "commands", + "reads_ops": "reads", + "transactions_ops": "transactions", + "writes_ops": "writes", }, - { - oldName: "mongodb_ss_opLatencies_latency", - prefix: "mongodb_ss_opLatencies", - suffixLabel: "op_type", - suffixMapping: map[string]string{ - "commands_latency": "commands", - "reads_latency": "reads", - "transactions_latency": "transactions", - "writes_latency": "writes", - }, + }, + { + oldName: "mongodb_ss_opLatencies_latency", + prefix: "mongodb_ss_opLatencies", + suffixLabel: "op_type", + suffixMapping: map[string]string{ + "commands_latency": "commands", + "reads_latency": "reads", + "transactions_latency": "transactions", + "writes_latency": "writes", }, - // mongodb_ss_wt_concurrentTransactions_read_out - // mongodb_ss_wt_concurrentTransactions_write_out - { - oldName: "mongodb_ss_wt_concurrentTransactions_out", - prefix: "mongodb_ss_wt_concurrentTransactions", - suffixLabel: "txn_rw", - suffixMapping: map[string]string{ - "read_out": "read", - "write_out": "write", - }, + }, + // mongodb_ss_wt_concurrentTransactions_read_out + // mongodb_ss_wt_concurrentTransactions_write_out + { + oldName: "mongodb_ss_wt_concurrentTransactions_out", + prefix: "mongodb_ss_wt_concurrentTransactions", + suffixLabel: "txn_rw", + suffixMapping: map[string]string{ + "read_out": "read", + "write_out": "write", }, - // mongodb_ss_wt_concurrentTransactions_read_available - // mongodb_ss_wt_concurrentTransactions_write_available - { - oldName: "mongodb_ss_wt_concurrentTransactions_available", - prefix: "mongodb_ss_wt_concurrentTransactions", - suffixLabel: "txn_rw", - suffixMapping: map[string]string{ - "read_available": "read", - "write_available": "write", - }, + }, + // mongodb_ss_wt_concurrentTransactions_read_available + // mongodb_ss_wt_concurrentTransactions_write_available + { + oldName: "mongodb_ss_wt_concurrentTransactions_available", + prefix: "mongodb_ss_wt_concurrentTransactions", + suffixLabel: "txn_rw", + suffixMapping: map[string]string{ + "read_available": "read", + "write_available": "write", }, - // mongodb_ss_wt_concurrentTransactions_read_totalTickets - // mongodb_ss_wt_concurrentTransactions_write_totalTickets - { - oldName: "mongodb_ss_wt_concurrentTransactions_totalTickets", - prefix: "mongodb_ss_wt_concurrentTransactions", - suffixLabel: "txn_rw", - suffixMapping: map[string]string{ - "read_totalTickets": "read", - "write_totalTickets": "write", - }, + }, + // mongodb_ss_wt_concurrentTransactions_read_totalTickets + // mongodb_ss_wt_concurrentTransactions_write_totalTickets + { + oldName: "mongodb_ss_wt_concurrentTransactions_totalTickets", + prefix: "mongodb_ss_wt_concurrentTransactions", + suffixLabel: "txn_rw", + suffixMapping: map[string]string{ + "read_totalTickets": "read", + "write_totalTickets": "write", }, - } + }, } diff --git a/exporter/metrics_test.go b/exporter/metrics_test.go index da0a00743..1a20591b6 100644 --- a/exporter/metrics_test.go +++ b/exporter/metrics_test.go @@ -211,7 +211,7 @@ func TestRawToCompatibleRawMetric(t *testing.T) { } for _, tc := range testCases { - m := metricRenameAndLabel(tc.in, specialConversions()) + m := metricRenameAndLabel(tc.in, specialConversions) assert.Equal(t, m[0], tc.want) } } diff --git a/exporter/top_collector.go b/exporter/top_collector.go index cd99023e9..6a9d09bea 100644 --- a/exporter/top_collector.go +++ b/exporter/top_collector.go @@ -42,7 +42,7 @@ func newTopCollector(ctx context.Context, client *mongo.Client, logger *logrus.L return &topCollector{ ctx: ctx, base: newBaseCollector(client, logger.WithFields(logrus.Fields{"collector": "top"})), - compatibleMode: compatible, + compatibleMode: false, // there are no compatible metrics for this collector. topologyInfo: topology, } } diff --git a/exporter/v1_compatibility.go b/exporter/v1_compatibility.go index 29fc2c7b5..85016069f 100644 --- a/exporter/v1_compatibility.go +++ b/exporter/v1_compatibility.go @@ -211,7 +211,7 @@ func sumMetrics(m bson.M, paths [][]string) (float64, error) { // Converts new metric to the old metric style and append it to the response slice. func appendCompatibleMetric(res []prometheus.Metric, rm *rawMetric) []prometheus.Metric { - compatibleMetrics := metricRenameAndLabel(rm, conversions()) + compatibleMetrics := metricRenameAndLabel(rm, conversions) if compatibleMetrics == nil { return res } @@ -230,451 +230,448 @@ func appendCompatibleMetric(res []prometheus.Metric, rm *rawMetric) []prometheus return res } -//nolint:funlen -func conversions() []conversion { - return []conversion{ - { - oldName: "mongodb_asserts_total", - newName: "mongodb_ss_asserts", - labelConversions: map[string]string{"assert_type": "type"}, - }, - { - oldName: "mongodb_connections", - newName: "mongodb_ss_connections", - labelConversions: map[string]string{"conn_type": "state"}, - }, - { - oldName: "mongodb_connections_metrics_created_total", - newName: "mongodb_ss_connections_totalCreated", - }, - { - oldName: "mongodb_extra_info_page_faults_total", - newName: "mongodb_ss_extra_info_page_faults", - }, - { - oldName: "mongodb_mongod_durability_journaled_megabytes", - newName: "mongodb_ss_dur_journaledMB", - }, - { - oldName: "mongodb_mongod_durability_commits", - newName: "mongodb_ss_dur_commits", - }, - { - oldName: "mongodb_mongod_background_flushing_average_milliseconds", - newName: "mongodb_ss_backgroundFlushing_average_ms", - }, - { - oldName: "mongodb_mongod_global_lock_client", - prefix: "mongodb_ss_globalLock_activeClients", - suffixLabel: "type", - suffixMapping: map[string]string{ - "readers": "reader", - "writers": "writer", - "total": "total", - }, - }, - { - oldName: "mongodb_mongod_global_lock_current_queue", - newName: "mongodb_ss_globalLock_currentQueue", - labelConversions: map[string]string{"count_type": "type"}, - labelValueConversions: map[string]string{ - "readers": "reader", - "writers": "writer", - }, - }, - { - oldName: "mongodb_instance_local_time", - newName: "mongodb_start", - }, - - { - oldName: "mongodb_mongod_instance_uptime_seconds", - newName: "mongodb_ss_uptime", - }, - { - oldName: "mongodb_instance_uptime_seconds", - newName: "mongodb_ss_uptime", - }, - { - oldName: "mongodb_mongod_locks_time_locked_local_microseconds_total", - newName: "mongodb_ss_locks_Local_acquireCount_[rw]", - }, - { - oldName: "mongodb_memory", - newName: "mongodb_ss_mem_[resident|virtual]", - }, - { - oldName: "mongodb_memory", - prefix: "mongodb_ss_mem", - suffixLabel: "type", - suffixMapping: map[string]string{ - "mapped": "mapped", - "mappedWithJournal": "mapped_with_journal", - }, - }, - { - oldName: "mongodb_mongod_metrics_cursor_open", - newName: "mongodb_ss_metrics_cursor_open", - labelConversions: map[string]string{"csr_type": "state"}, - }, - { - oldName: "mongodb_mongod_metrics_cursor_timed_out_total", - newName: "mongodb_ss_metrics_cursor_timedOut", - }, - { - oldName: "mongodb_mongod_metrics_document_total", - newName: "mongodb_ss_metric_document", - labelConversions: map[string]string{"doc_op_type": "type"}, - }, - { - oldName: "mongodb_mongod_metrics_get_last_error_wtime_num_total", - newName: "mongodb_ss_metrics_getLastError_wtime_num", - }, - { - oldName: "mongodb_mongod_metrics_get_last_error_wtimeouts_total", - newName: "mongodb_ss_metrics_getLastError_wtimeouts", - }, - { - oldName: "mongodb_mongod_metrics_operation_total", - prefix: "mongodb_ss_metrics_operation", - suffixLabel: "state", - suffixMapping: map[string]string{ - "scanAndOrder": "scanAndOrder", - "writeConflicts": "writeConflicts", - }, - }, - { - oldName: "mongodb_mongod_metrics_query_executor_total", - prefix: "mongodb_ss_metrics_query", - suffixLabel: "state", - }, - { - oldName: "mongodb_mongod_metrics_record_moves_total", - newName: "mongodb_ss_metrics_record_moves", - }, - { - oldName: "mongodb_mongod_metrics_repl_apply_batches_num_total", - newName: "mongodb_ss_metrics_repl_apply_batches_num", - }, - { - oldName: "mongodb_mongod_metrics_repl_apply_batches_total_milliseconds", - newName: "mongodb_ss_metrics_repl_apply_batches_totalMillis", - }, - { - oldName: "mongodb_mongod_metrics_repl_apply_ops_total", - newName: "mongodb_ss_metrics_repl_apply_ops", - }, - { - oldName: "mongodb_mongod_metrics_repl_buffer_count", - newName: "mongodb_ss_metrics_repl_buffer_count", - }, - { - oldName: "mongodb_mongod_metrics_repl_buffer_max_size_bytes", - newName: "mongodb_ss_metrics_repl_buffer_maxSizeBytes", - }, - { - oldName: "mongodb_mongod_metrics_repl_buffer_size_bytes", - newName: "mongodb_ss_metrics_repl_buffer_sizeBytes", - }, - { - oldName: "mongodb_mongod_metrics_repl_executor_queue", - prefix: "mongodb_ss_metrics_repl_executor_queues", - suffixLabel: "type", - }, - { - oldName: "mongodb_mongod_metrics_repl_executor_unsignaled_events", - newName: "mongodb_ss_metrics_repl_executor_unsignaledEvents", - }, - { - oldName: "mongodb_mongod_metrics_repl_network_bytes_total", - newName: "mongodb_ss_metrics_repl_network_bytes", - }, - { - oldName: "mongodb_mongod_metrics_repl_network_getmores_num_total", - newName: "mongodb_ss_metrics_repl_network_getmores_num", - }, - { - oldName: "mongodb_mongod_metrics_repl_network_getmores_total_milliseconds", - newName: "mongodb_ss_metrics_repl_network_getmores_totalMillis", - }, - { - oldName: "mongodb_mongod_metrics_repl_network_ops_total", - newName: "mongodb_ss_metrics_repl_network_ops", - }, - { - oldName: "mongodb_mongod_metrics_repl_network_readers_created_total", - newName: "mongodb_ss_metrics_repl_network_readersCreated", - }, - { - oldName: "mongodb_mongod_metrics_ttl_deleted_documents_total", - newName: "mongodb_ss_metrics_ttl_deletedDocuments", - }, - { - oldName: "mongodb_mongod_metrics_ttl_passes_total", - newName: "mongodb_ss_metrics_ttl_passes", - }, - { - oldName: "mongodb_network_bytes_total", - prefix: "mongodb_ss_network", - suffixLabel: "state", - }, - { - oldName: "mongodb_network_metrics_num_requests_total", - newName: "mongodb_ss_network_numRequests", - }, - { - oldName: "mongodb_mongod_op_counters_repl_total", - newName: "mongodb_ss_opcountersRepl", - labelConversions: map[string]string{"legacy_op_type": "type"}, - }, - { - oldName: "mongodb_op_counters_total", - newName: "mongodb_ss_opcounters", - labelConversions: map[string]string{"legacy_op_type": "type"}, - }, - { - oldName: "mongodb_mongod_wiredtiger_blockmanager_blocks_total", - prefix: "mongodb_ss_wt_block_manager", - suffixLabel: "type", - }, - { - oldName: "mongodb_mongod_wiredtiger_cache_max_bytes", - newName: "mongodb_ss_wt_cache_maximum_bytes_configured", - }, - { - oldName: "mongodb_mongod_wiredtiger_cache_overhead_percent", - newName: "mongodb_ss_wt_cache_percentage_overhead", - }, - { - oldName: "mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets", - newName: "mongodb_ss_wt_concurrentTransactions_available", - }, - { - oldName: "mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets", - newName: "mongodb_ss_wt_concurrentTransactions_out", - }, - { - oldName: "mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets", - newName: "mongodb_ss_wt_concurrentTransactions_totalTickets", - }, - { - oldName: "mongodb_mongod_wiredtiger_log_records_scanned_total", - newName: "mongodb_ss_wt_log_records_processed_by_log_scan", - }, - { - oldName: "mongodb_mongod_wiredtiger_session_open_cursors_total", - newName: "mongodb_ss_wt_session_open_cursor_count", - }, - { - oldName: "mongodb_mongod_wiredtiger_session_open_sessions_total", - newName: "mongodb_ss_wt_session_open_session_count", - }, - { - oldName: "mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds_total", - newName: "mongodb_ss_wt_txn_transaction_checkpoint_total_time_msecs", - }, - { - oldName: "mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds_total", - newName: "mongodb_ss_wt_checkpoint_total_time_msecs", - }, - { - oldName: "mongodb_mongod_wiredtiger_transactions_running_checkpoints", - newName: "mongodb_ss_wt_txn_transaction_checkpoint_currently_running", - }, - { - oldName: "mongodb_mongod_wiredtiger_transactions_running_checkpoints", - newName: "mongodb_ss_wt_checkpoint_currently_running", - }, - { - oldName: "mongodb_ss_tcmalloc_tcmalloc_thread_cache_free_bytes", - newName: "mongodb_ss_tcmalloc_tcmalloc_thread_cache_free", - }, - { - oldName: "mongodb_mongod_wiredtiger_transactions_total", - prefix: "mongodb_ss_wt_txn_transactions", - suffixLabel: "type", - suffixMapping: map[string]string{ - "begins": "begins", - "checkpoints": "checkpoints", - "committed": "committed", - "rolled_back": "rolled_back", - }, - }, - { - oldName: "mongodb_mongod_wiredtiger_blockmanager_bytes_total", - prefix: "mongodb_ss_wt_block_manager", - suffixLabel: "type", - suffixMapping: map[string]string{ - "bytes_read": "read", "mapped_bytes_read": "read_mapped", - "bytes_written": "written", - }, - }, - // the 2 metrics bellow have the same prefix. - { - oldName: "mongodb_mongod_wiredtiger_cache_bytes", - prefix: "mongodb_ss_wt_cache_bytes", - suffixLabel: "type", - suffixMapping: map[string]string{ - "currently_in_the_cache": "total", - "tracked_dirty_bytes_in_the_cache": "dirty", - "tracked_bytes_belonging_to_internal_pages_in_the_cache": " internal_pages", - "tracked_bytes_belonging_to_leaf_pages_in_the_cache": "internal_pages", - }, - }, - { - oldName: "mongodb_mongod_wiredtiger_cache_bytes_total", - prefix: "mongodb_ss_wt_cache", - suffixLabel: "type", - suffixMapping: map[string]string{ - "bytes_read_into_cache": "read", - "bytes_written_from_cache": "written", - }, - }, - { - oldName: "mongodb_mongod_wiredtiger_cache_pages", - prefix: "mongodb_ss_wt_cache", - suffixLabel: "type", - suffixMapping: map[string]string{ - "pages_currently_held_in_the_cache": "total", - "tracked_dirty_pages_in_the_cache": "dirty", - }, - }, - { - oldName: "mongodb_mongod_wiredtiger_cache_pages_total", - prefix: "mongodb_ss_wt_cache", - suffixLabel: "type", - suffixMapping: map[string]string{ - "pages_read_into_cache": "read", - "pages_written_from_cache": "written", - }, - }, - { - oldName: "mongodb_mongod_wiredtiger_log_records_total", - prefix: "mongodb_ss_wt_log", - suffixLabel: "type", - suffixMapping: map[string]string{ - "log_records_compressed": "compressed", - "log_records_not_compressed": "uncompressed", - }, - }, - { - oldName: "mongodb_mongod_wiredtiger_log_bytes_total", - prefix: "mongodb_ss_wt_log", - suffixLabel: "type", - suffixMapping: map[string]string{ - "log_bytes_of_payload_data": "payload", - "log_bytes_written": "unwritten", - }, - }, - { - oldName: "mongodb_mongod_wiredtiger_log_operations_total", - prefix: "mongodb_ss_wt_log", - suffixLabel: "type", - suffixMapping: map[string]string{ - "log_read_operations": "read", - "log_write_operations": "write", - "log_scan_operations": "scan", - "log_scan_records_requiring_two_reads": "scan_double", - "log_sync_operations": "sync", - "log_sync_dir_operations": "sync_dir", - "log_flush_operations": "flush", - }, - }, - { - oldName: "mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds", - prefix: "mongodb_ss_wt_txn_transaction_checkpoint", - suffixLabel: "type", - suffixMapping: map[string]string{ - "min_time_msecs": "min", - "max_time_msecs": "max", - }, - }, - { - oldName: "mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds", - prefix: "mongodb_ss_wt_checkpoint", - suffixLabel: "type", - suffixMapping: map[string]string{ - "min_time_msecs": "min", - "max_time_msecs": "max", - }, - }, - { - oldName: "mongodb_mongod_global_lock_current_queue", - prefix: "mongodb_mongod_global_lock_current_queue", - labelConversions: map[string]string{"op_type": "type"}, - }, - { - oldName: "mongodb_mongod_op_latencies_ops_total", - newName: "mongodb_ss_opLatencies_ops", - labelConversions: map[string]string{"op_type": "type"}, - labelValueConversions: map[string]string{ - "commands": "command", - "reads": "read", - "writes": "write", - }, - }, - { - oldName: "mongodb_mongod_op_latencies_latency_total", - newName: "mongodb_ss_opLatencies_latency", - labelConversions: map[string]string{"op_type": "type"}, - labelValueConversions: map[string]string{ - "commands": "command", - "reads": "read", - "writes": "write", - }, - }, - { - oldName: "mongodb_mongod_metrics_document_total", - newName: "mongodb_ss_metrics_document", - labelConversions: map[string]string{"doc_op_type": "state"}, - }, - { - oldName: "mongodb_mongod_metrics_query_executor_total", - prefix: "mongodb_ss_metrics_queryExecutor", - suffixLabel: "state", - suffixMapping: map[string]string{ - "scanned": "scanned", - "scannedObjects": "scanned_objects", - }, - }, - { - oldName: "mongodb_memory", - prefix: "mongodb_ss_mem", - suffixLabel: "type", - suffixMapping: map[string]string{ - "resident": "resident", - "virtual": "virtual", - }, - }, - { - oldName: "mongodb_mongod_metrics_get_last_error_wtime_total_milliseconds", - newName: "mongodb_ss_metrics_getLastError_wtime_totalMillis", - }, - { - oldName: "mongodb_ss_wt_cache_maximum_bytes_configured", - newName: "mongodb_mongod_wiredtiger_cache_max_bytes", - }, - { - oldName: "mongodb_mongod_db_collections_total", - newName: "mongodb_dbstats_collections", - }, - { - oldName: "mongodb_mongod_db_data_size_bytes", - newName: "mongodb_dbstats_dataSize", - }, - { - oldName: "mongodb_mongod_db_index_size_bytes", - newName: "mongodb_dbstats_indexSize", - }, - { - oldName: "mongodb_mongod_db_indexes_total", - newName: "mongodb_dbstats_indexes", - }, - { - oldName: "mongodb_mongod_db_objects_total", - newName: "mongodb_dbstats_objects", - }, - } +var conversions = []conversion{ + { + oldName: "mongodb_asserts_total", + newName: "mongodb_ss_asserts", + labelConversions: map[string]string{"assert_type": "type"}, + }, + { + oldName: "mongodb_connections", + newName: "mongodb_ss_connections", + labelConversions: map[string]string{"conn_type": "state"}, + }, + { + oldName: "mongodb_connections_metrics_created_total", + newName: "mongodb_ss_connections_totalCreated", + }, + { + oldName: "mongodb_extra_info_page_faults_total", + newName: "mongodb_ss_extra_info_page_faults", + }, + { + oldName: "mongodb_mongod_durability_journaled_megabytes", + newName: "mongodb_ss_dur_journaledMB", + }, + { + oldName: "mongodb_mongod_durability_commits", + newName: "mongodb_ss_dur_commits", + }, + { + oldName: "mongodb_mongod_background_flushing_average_milliseconds", + newName: "mongodb_ss_backgroundFlushing_average_ms", + }, + { + oldName: "mongodb_mongod_global_lock_client", + prefix: "mongodb_ss_globalLock_activeClients", + suffixLabel: "type", + suffixMapping: map[string]string{ + "readers": "reader", + "writers": "writer", + "total": "total", + }, + }, + { + oldName: "mongodb_mongod_global_lock_current_queue", + newName: "mongodb_ss_globalLock_currentQueue", + labelConversions: map[string]string{"count_type": "type"}, + labelValueConversions: map[string]string{ + "readers": "reader", + "writers": "writer", + }, + }, + { + oldName: "mongodb_instance_local_time", + newName: "mongodb_start", + }, + + { + oldName: "mongodb_mongod_instance_uptime_seconds", + newName: "mongodb_ss_uptime", + }, + { + oldName: "mongodb_instance_uptime_seconds", + newName: "mongodb_ss_uptime", + }, + { + oldName: "mongodb_mongod_locks_time_locked_local_microseconds_total", + newName: "mongodb_ss_locks_Local_acquireCount_[rw]", + }, + { + oldName: "mongodb_memory", + newName: "mongodb_ss_mem_[resident|virtual]", + }, + { + oldName: "mongodb_memory", + prefix: "mongodb_ss_mem", + suffixLabel: "type", + suffixMapping: map[string]string{ + "mapped": "mapped", + "mappedWithJournal": "mapped_with_journal", + }, + }, + { + oldName: "mongodb_mongod_metrics_cursor_open", + newName: "mongodb_ss_metrics_cursor_open", + labelConversions: map[string]string{"csr_type": "state"}, + }, + { + oldName: "mongodb_mongod_metrics_cursor_timed_out_total", + newName: "mongodb_ss_metrics_cursor_timedOut", + }, + { + oldName: "mongodb_mongod_metrics_document_total", + newName: "mongodb_ss_metric_document", + labelConversions: map[string]string{"doc_op_type": "type"}, + }, + { + oldName: "mongodb_mongod_metrics_get_last_error_wtime_num_total", + newName: "mongodb_ss_metrics_getLastError_wtime_num", + }, + { + oldName: "mongodb_mongod_metrics_get_last_error_wtimeouts_total", + newName: "mongodb_ss_metrics_getLastError_wtimeouts", + }, + { + oldName: "mongodb_mongod_metrics_operation_total", + prefix: "mongodb_ss_metrics_operation", + suffixLabel: "state", + suffixMapping: map[string]string{ + "scanAndOrder": "scanAndOrder", + "writeConflicts": "writeConflicts", + }, + }, + { + oldName: "mongodb_mongod_metrics_query_executor_total", + prefix: "mongodb_ss_metrics_query", + suffixLabel: "state", + }, + { + oldName: "mongodb_mongod_metrics_record_moves_total", + newName: "mongodb_ss_metrics_record_moves", + }, + { + oldName: "mongodb_mongod_metrics_repl_apply_batches_num_total", + newName: "mongodb_ss_metrics_repl_apply_batches_num", + }, + { + oldName: "mongodb_mongod_metrics_repl_apply_batches_total_milliseconds", + newName: "mongodb_ss_metrics_repl_apply_batches_totalMillis", + }, + { + oldName: "mongodb_mongod_metrics_repl_apply_ops_total", + newName: "mongodb_ss_metrics_repl_apply_ops", + }, + { + oldName: "mongodb_mongod_metrics_repl_buffer_count", + newName: "mongodb_ss_metrics_repl_buffer_count", + }, + { + oldName: "mongodb_mongod_metrics_repl_buffer_max_size_bytes", + newName: "mongodb_ss_metrics_repl_buffer_maxSizeBytes", + }, + { + oldName: "mongodb_mongod_metrics_repl_buffer_size_bytes", + newName: "mongodb_ss_metrics_repl_buffer_sizeBytes", + }, + { + oldName: "mongodb_mongod_metrics_repl_executor_queue", + prefix: "mongodb_ss_metrics_repl_executor_queues", + suffixLabel: "type", + }, + { + oldName: "mongodb_mongod_metrics_repl_executor_unsignaled_events", + newName: "mongodb_ss_metrics_repl_executor_unsignaledEvents", + }, + { + oldName: "mongodb_mongod_metrics_repl_network_bytes_total", + newName: "mongodb_ss_metrics_repl_network_bytes", + }, + { + oldName: "mongodb_mongod_metrics_repl_network_getmores_num_total", + newName: "mongodb_ss_metrics_repl_network_getmores_num", + }, + { + oldName: "mongodb_mongod_metrics_repl_network_getmores_total_milliseconds", + newName: "mongodb_ss_metrics_repl_network_getmores_totalMillis", + }, + { + oldName: "mongodb_mongod_metrics_repl_network_ops_total", + newName: "mongodb_ss_metrics_repl_network_ops", + }, + { + oldName: "mongodb_mongod_metrics_repl_network_readers_created_total", + newName: "mongodb_ss_metrics_repl_network_readersCreated", + }, + { + oldName: "mongodb_mongod_metrics_ttl_deleted_documents_total", + newName: "mongodb_ss_metrics_ttl_deletedDocuments", + }, + { + oldName: "mongodb_mongod_metrics_ttl_passes_total", + newName: "mongodb_ss_metrics_ttl_passes", + }, + { + oldName: "mongodb_network_bytes_total", + prefix: "mongodb_ss_network", + suffixLabel: "state", + }, + { + oldName: "mongodb_network_metrics_num_requests_total", + newName: "mongodb_ss_network_numRequests", + }, + { + oldName: "mongodb_mongod_op_counters_repl_total", + newName: "mongodb_ss_opcountersRepl", + labelConversions: map[string]string{"legacy_op_type": "type"}, + }, + { + oldName: "mongodb_op_counters_total", + newName: "mongodb_ss_opcounters", + labelConversions: map[string]string{"legacy_op_type": "type"}, + }, + { + oldName: "mongodb_mongod_wiredtiger_blockmanager_blocks_total", + prefix: "mongodb_ss_wt_block_manager", + suffixLabel: "type", + }, + { + oldName: "mongodb_mongod_wiredtiger_cache_max_bytes", + newName: "mongodb_ss_wt_cache_maximum_bytes_configured", + }, + { + oldName: "mongodb_mongod_wiredtiger_cache_overhead_percent", + newName: "mongodb_ss_wt_cache_percentage_overhead", + }, + { + oldName: "mongodb_mongod_wiredtiger_concurrent_transactions_available_tickets", + newName: "mongodb_ss_wt_concurrentTransactions_available", + }, + { + oldName: "mongodb_mongod_wiredtiger_concurrent_transactions_out_tickets", + newName: "mongodb_ss_wt_concurrentTransactions_out", + }, + { + oldName: "mongodb_mongod_wiredtiger_concurrent_transactions_total_tickets", + newName: "mongodb_ss_wt_concurrentTransactions_totalTickets", + }, + { + oldName: "mongodb_mongod_wiredtiger_log_records_scanned_total", + newName: "mongodb_ss_wt_log_records_processed_by_log_scan", + }, + { + oldName: "mongodb_mongod_wiredtiger_session_open_cursors_total", + newName: "mongodb_ss_wt_session_open_cursor_count", + }, + { + oldName: "mongodb_mongod_wiredtiger_session_open_sessions_total", + newName: "mongodb_ss_wt_session_open_session_count", + }, + { + oldName: "mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds_total", + newName: "mongodb_ss_wt_txn_transaction_checkpoint_total_time_msecs", + }, + { + oldName: "mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds_total", + newName: "mongodb_ss_wt_checkpoint_total_time_msecs", + }, + { + oldName: "mongodb_mongod_wiredtiger_transactions_running_checkpoints", + newName: "mongodb_ss_wt_txn_transaction_checkpoint_currently_running", + }, + { + oldName: "mongodb_mongod_wiredtiger_transactions_running_checkpoints", + newName: "mongodb_ss_wt_checkpoint_currently_running", + }, + { + oldName: "mongodb_ss_tcmalloc_tcmalloc_thread_cache_free_bytes", + newName: "mongodb_ss_tcmalloc_tcmalloc_thread_cache_free", + }, + { + oldName: "mongodb_mongod_wiredtiger_transactions_total", + prefix: "mongodb_ss_wt_txn_transactions", + suffixLabel: "type", + suffixMapping: map[string]string{ + "begins": "begins", + "checkpoints": "checkpoints", + "committed": "committed", + "rolled_back": "rolled_back", + }, + }, + { + oldName: "mongodb_mongod_wiredtiger_blockmanager_bytes_total", + prefix: "mongodb_ss_wt_block_manager", + suffixLabel: "type", + suffixMapping: map[string]string{ + "bytes_read": "read", "mapped_bytes_read": "read_mapped", + "bytes_written": "written", + }, + }, + // the 2 metrics bellow have the same prefix. + { + oldName: "mongodb_mongod_wiredtiger_cache_bytes", + prefix: "mongodb_ss_wt_cache_bytes", + suffixLabel: "type", + suffixMapping: map[string]string{ + "currently_in_the_cache": "total", + "tracked_dirty_bytes_in_the_cache": "dirty", + "tracked_bytes_belonging_to_internal_pages_in_the_cache": " internal_pages", + "tracked_bytes_belonging_to_leaf_pages_in_the_cache": "internal_pages", + }, + }, + { + oldName: "mongodb_mongod_wiredtiger_cache_bytes_total", + prefix: "mongodb_ss_wt_cache", + suffixLabel: "type", + suffixMapping: map[string]string{ + "bytes_read_into_cache": "read", + "bytes_written_from_cache": "written", + }, + }, + { + oldName: "mongodb_mongod_wiredtiger_cache_pages", + prefix: "mongodb_ss_wt_cache", + suffixLabel: "type", + suffixMapping: map[string]string{ + "pages_currently_held_in_the_cache": "total", + "tracked_dirty_pages_in_the_cache": "dirty", + }, + }, + { + oldName: "mongodb_mongod_wiredtiger_cache_pages_total", + prefix: "mongodb_ss_wt_cache", + suffixLabel: "type", + suffixMapping: map[string]string{ + "pages_read_into_cache": "read", + "pages_written_from_cache": "written", + }, + }, + { + oldName: "mongodb_mongod_wiredtiger_log_records_total", + prefix: "mongodb_ss_wt_log", + suffixLabel: "type", + suffixMapping: map[string]string{ + "log_records_compressed": "compressed", + "log_records_not_compressed": "uncompressed", + }, + }, + { + oldName: "mongodb_mongod_wiredtiger_log_bytes_total", + prefix: "mongodb_ss_wt_log", + suffixLabel: "type", + suffixMapping: map[string]string{ + "log_bytes_of_payload_data": "payload", + "log_bytes_written": "unwritten", + }, + }, + { + oldName: "mongodb_mongod_wiredtiger_log_operations_total", + prefix: "mongodb_ss_wt_log", + suffixLabel: "type", + suffixMapping: map[string]string{ + "log_read_operations": "read", + "log_write_operations": "write", + "log_scan_operations": "scan", + "log_scan_records_requiring_two_reads": "scan_double", + "log_sync_operations": "sync", + "log_sync_dir_operations": "sync_dir", + "log_flush_operations": "flush", + }, + }, + { + oldName: "mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds", + prefix: "mongodb_ss_wt_txn_transaction_checkpoint", + suffixLabel: "type", + suffixMapping: map[string]string{ + "min_time_msecs": "min", + "max_time_msecs": "max", + }, + }, + { + oldName: "mongodb_mongod_wiredtiger_transactions_checkpoint_milliseconds", + prefix: "mongodb_ss_wt_checkpoint", + suffixLabel: "type", + suffixMapping: map[string]string{ + "min_time_msecs": "min", + "max_time_msecs": "max", + }, + }, + { + oldName: "mongodb_mongod_global_lock_current_queue", + prefix: "mongodb_mongod_global_lock_current_queue", + labelConversions: map[string]string{"op_type": "type"}, + }, + { + oldName: "mongodb_mongod_op_latencies_ops_total", + newName: "mongodb_ss_opLatencies_ops", + labelConversions: map[string]string{"op_type": "type"}, + labelValueConversions: map[string]string{ + "commands": "command", + "reads": "read", + "writes": "write", + }, + }, + { + oldName: "mongodb_mongod_op_latencies_latency_total", + newName: "mongodb_ss_opLatencies_latency", + labelConversions: map[string]string{"op_type": "type"}, + labelValueConversions: map[string]string{ + "commands": "command", + "reads": "read", + "writes": "write", + }, + }, + { + oldName: "mongodb_mongod_metrics_document_total", + newName: "mongodb_ss_metrics_document", + labelConversions: map[string]string{"doc_op_type": "state"}, + }, + { + oldName: "mongodb_mongod_metrics_query_executor_total", + prefix: "mongodb_ss_metrics_queryExecutor", + suffixLabel: "state", + suffixMapping: map[string]string{ + "scanned": "scanned", + "scannedObjects": "scanned_objects", + }, + }, + { + oldName: "mongodb_memory", + prefix: "mongodb_ss_mem", + suffixLabel: "type", + suffixMapping: map[string]string{ + "resident": "resident", + "virtual": "virtual", + }, + }, + { + oldName: "mongodb_mongod_metrics_get_last_error_wtime_total_milliseconds", + newName: "mongodb_ss_metrics_getLastError_wtime_totalMillis", + }, + { + oldName: "mongodb_ss_wt_cache_maximum_bytes_configured", + newName: "mongodb_mongod_wiredtiger_cache_max_bytes", + }, + { + oldName: "mongodb_mongod_db_collections_total", + newName: "mongodb_dbstats_collections", + }, + { + oldName: "mongodb_mongod_db_data_size_bytes", + newName: "mongodb_dbstats_dataSize", + }, + { + oldName: "mongodb_mongod_db_index_size_bytes", + newName: "mongodb_dbstats_indexSize", + }, + { + oldName: "mongodb_mongod_db_indexes_total", + newName: "mongodb_dbstats_indexes", + }, + { + oldName: "mongodb_mongod_db_objects_total", + newName: "mongodb_dbstats_objects", + }, } // Third metric renaming case (3). @@ -688,49 +685,47 @@ type lockMetric struct { labels map[string]string } -func lockMetrics() []lockMetric { - return []lockMetric{ - { - name: "mongodb_ss_locks_acquireCount", - path: []string{"serverStatus", "locks", "ParallelBatchWriterMode", "acquireCount", "r"}, - labels: map[string]string{"lock_mode": "r", "resource": "ParallelBatchWriterMode"}, - }, - { - name: "mongodb_ss_locks_acquireCount", - path: []string{"serverStatus", "locks", "ParallelBatchWriterMode", "acquireCount", "w"}, - labels: map[string]string{"lock_mode": "w", "resource": "ReplicationStateTransition"}, - }, - { - name: "mongodb_ss_locks_acquireCount", - path: []string{"serverStatus", "locks", "ReplicationStateTransition", "acquireCount", "w"}, - labels: map[string]string{"resource": "ReplicationStateTransition", "lock_mode": "w"}, - }, - { - name: "mongodb_ss_locks_acquireWaitCount", - path: []string{"serverStatus", "locks", "ReplicationStateTransition", "acquireCount", "W"}, - labels: map[string]string{"lock_mode": "W", "resource": "ReplicationStateTransition"}, - }, - { - name: "mongodb_ss_locks_timeAcquiringMicros", - path: []string{"serverStatus", "locks", "ReplicationStateTransition", "timeAcquiringMicros", "w"}, - labels: map[string]string{"lock_mode": "w", "resource": "ReplicationStateTransition"}, - }, - { - name: "mongodb_ss_locks_acquireCount", - path: []string{"serverStatus", "locks", "Global", "acquireCount", "r"}, - labels: map[string]string{"lock_mode": "r", "resource": "Global"}, - }, - { - name: "mongodb_ss_locks_acquireCount", - path: []string{"serverStatus", "locks", "Global", "acquireCount", "w"}, - labels: map[string]string{"lock_mode": "w", "resource": "Global"}, - }, - { - name: "mongodb_ss_locks_acquireCount", - path: []string{"serverStatus", "locks", "Global", "acquireCount", "W"}, - labels: map[string]string{"lock_mode": "W", "resource": "Global"}, - }, - } +var lockMetrics = []lockMetric{ + { + name: "mongodb_ss_locks_acquireCount", + path: []string{"serverStatus", "locks", "ParallelBatchWriterMode", "acquireCount", "r"}, + labels: map[string]string{"lock_mode": "r", "resource": "ParallelBatchWriterMode"}, + }, + { + name: "mongodb_ss_locks_acquireCount", + path: []string{"serverStatus", "locks", "ParallelBatchWriterMode", "acquireCount", "w"}, + labels: map[string]string{"lock_mode": "w", "resource": "ReplicationStateTransition"}, + }, + { + name: "mongodb_ss_locks_acquireCount", + path: []string{"serverStatus", "locks", "ReplicationStateTransition", "acquireCount", "w"}, + labels: map[string]string{"resource": "ReplicationStateTransition", "lock_mode": "w"}, + }, + { + name: "mongodb_ss_locks_acquireWaitCount", + path: []string{"serverStatus", "locks", "ReplicationStateTransition", "acquireCount", "W"}, + labels: map[string]string{"lock_mode": "W", "resource": "ReplicationStateTransition"}, + }, + { + name: "mongodb_ss_locks_timeAcquiringMicros", + path: []string{"serverStatus", "locks", "ReplicationStateTransition", "timeAcquiringMicros", "w"}, + labels: map[string]string{"lock_mode": "w", "resource": "ReplicationStateTransition"}, + }, + { + name: "mongodb_ss_locks_acquireCount", + path: []string{"serverStatus", "locks", "Global", "acquireCount", "r"}, + labels: map[string]string{"lock_mode": "r", "resource": "Global"}, + }, + { + name: "mongodb_ss_locks_acquireCount", + path: []string{"serverStatus", "locks", "Global", "acquireCount", "w"}, + labels: map[string]string{"lock_mode": "w", "resource": "Global"}, + }, + { + name: "mongodb_ss_locks_acquireCount", + path: []string{"serverStatus", "locks", "Global", "acquireCount", "W"}, + labels: map[string]string{"lock_mode": "W", "resource": "Global"}, + }, } // locksMetrics returns the list of lock metrics as a prometheus.Metric slice @@ -738,10 +733,9 @@ func lockMetrics() []lockMetric { // ready to be exposed, taking the value for each metric from th provided bson.M structure from // getDiagnosticData. func locksMetrics(logger *logrus.Entry, m bson.M) []prometheus.Metric { - metrics := lockMetrics() - res := make([]prometheus.Metric, 0, len(metrics)) + res := make([]prometheus.Metric, 0, len(lockMetrics)) - for _, lm := range metrics { + for _, lm := range lockMetrics { mm, err := makeLockMetric(m, lm) if mm == nil { continue @@ -790,23 +784,21 @@ type specialMetric struct { help string } -func specialMetricDefinitions() []specialMetric { - return []specialMetric{ - { - name: "mongodb_mongod_locks_time_acquiring_global_microseconds_total", - help: "sum of serverStatus.locks.Global.timeAcquiringMicros.[r|w]", - paths: [][]string{ - {"serverStatus", "locks", "Global", "timeAcquiringMicros", "r"}, - {"serverStatus", "locks", "Global", "timeAcquiringMicros", "w"}, - }, +var specialMetricDefinitions = []specialMetric{ + { + name: "mongodb_mongod_locks_time_acquiring_global_microseconds_total", + help: "sum of serverStatus.locks.Global.timeAcquiringMicros.[r|w]", + paths: [][]string{ + {"serverStatus", "locks", "Global", "timeAcquiringMicros", "r"}, + {"serverStatus", "locks", "Global", "timeAcquiringMicros", "w"}, }, - } + }, } func specialMetrics(ctx context.Context, client *mongo.Client, m bson.M, nodeType mongoDBNodeType, l *logrus.Entry) []prometheus.Metric { metrics := make([]prometheus.Metric, 0) - for _, def := range specialMetricDefinitions() { + for _, def := range specialMetricDefinitions { val, err := sumMetrics(m, def.paths) if err != nil { l.Errorf("cannot create metric for path: %v: %s", def.paths, err)