[BugFix] Add more check and info when compaction find row size incons…

…istency (#33972) Add more debug&verify log to address #33971 Signed-off-by: Binglin Chang <[email protected]> (cherry picked from commit 5322894) # Conflicts: # be/src/storage/tablet_updates.cpp
StarRocks · Oct 31, 2023 · dd9a0d6 · dd9a0d6
1 parent c588ad4
commit dd9a0d6
Show file tree

Hide file tree

Showing 3 changed files with 74 additions and 1 deletion.
diff --git a/be/src/storage/tablet_updates.cpp b/be/src/storage/tablet_updates.cpp
@@ -1762,12 +1762,51 @@ void TabletUpdates::_apply_compaction_commit(const EditVersionInfo& version_info
               << Substitute("($0/$1/$2)", t_load - t_start, t_index_delvec - t_load, t_write - t_index_delvec);
     VLOG(1) << "update compaction apply " << _debug_string(true, true);
     if (row_before != row_after) {
+<<<<<<< HEAD
         string msg = Substitute("actual row size changed after compaction $0 -> $1", row_before, row_after);
+=======
+        string msg = strings::Substitute(
+                "actual row size changed after compaction $0 -> $1 inputs:$2 output:$3 max_rowset_id:$4 "
+                "max_src_rssid:$5 $6",
+                row_before, row_after, PrettyPrinter::print_unique_int_list_range(info->inputs), rowset_id,
+                max_rowset_id, max_src_rssid, _debug_compaction_stats(info->inputs, rowset_id));
+>>>>>>> 5322894e53 ([BugFix] Add more check and info when compaction find row size inconsistency (#33972))
         LOG(ERROR) << msg << debug_string();
         _set_error(msg + _debug_version_info(true));
+        CHECK(output_rowset->verify().ok()) << msg;
     }
 }
 
+std::string TabletUpdates::_debug_compaction_stats(const std::vector<uint32_t>& input_rowsets,
+                                                   const uint32_t output_rowset) {
+    std::stringstream ss;
+    std::lock_guard lg(_rowset_stats_lock);
+    ss << "inputs:";
+    for (auto rowset_id : input_rowsets) {
+        auto iter = _rowset_stats.find(rowset_id);
+        if (iter == _rowset_stats.end()) {
+            ss << rowset_id << ":"
+               << "NA";
+        } else {
+            ss << rowset_id << ":" << iter->second->num_dels << "/" << iter->second->num_rows;
+        }
+        ss << " ";
+    }
+    ss << "output:";
+    auto iter = _rowset_stats.find(output_rowset);
+    if (iter == _rowset_stats.end()) {
+        ss << output_rowset << ":"
+           << "NA";
+    } else {
+        ss << output_rowset << ":" << iter->second->num_dels << "/" << iter->second->num_rows;
+    }
+    auto rs = _get_rowset(output_rowset);
+    if (rs) {
+        ss << " " << rs->unique_id();
+    }
+    return ss.str();
+}
+
 void TabletUpdates::to_updates_pb(TabletUpdatesPB* updates_pb) const {
     std::lock_guard rl(_lock);
     _to_updates_pb_unlocked(updates_pb);
@@ -3557,7 +3596,9 @@ Status TabletUpdates::load_snapshot(const SnapshotMeta& snapshot_meta, bool rest
             return Status::Cancelled("snapshot version too small");
         }
 
+        std::stringstream ss;
         uint32_t new_next_rowset_id = _next_rowset_id;
+        ss << "next_rowset_id before:" << _next_rowset_id << " rowsets:";
         for (const auto& rowset_meta_pb : snapshot_meta.rowset_metas()) {
             auto rowset_meta = std::make_shared<RowsetMeta>(rowset_meta_pb);
             const auto new_id = rowset_meta_pb.rowset_seg_id() + _next_rowset_id;
@@ -3567,6 +3608,7 @@ Status TabletUpdates::load_snapshot(const SnapshotMeta& snapshot_meta, bool rest
             RowsetSharedPtr* rowset = &new_rowsets[new_id];
             RETURN_IF_ERROR(RowsetFactory::create_rowset(&_tablet.tablet_schema(), _tablet.schema_hash_path(),
                                                          rowset_meta, rowset));
+            ss << new_id << ",";
             VLOG(2) << "add a new rowset " << tablet_id << "@" << new_id << "@" << rowset_meta->rowset_id();
         }
 
@@ -3590,18 +3632,21 @@ Status TabletUpdates::load_snapshot(const SnapshotMeta& snapshot_meta, bool rest
         }
         DCHECK_EQ(1, _edit_version_infos.size());
 
+        ss << " delvec:";
         WriteBatch wb;
         CHECK_FAIL(TabletMetaManager::clear_log(data_store, &wb, tablet_id));
         for (const auto& [rssid, delvec] : snapshot_meta.delete_vectors()) {
             auto id = rssid + _next_rowset_id;
             CHECK_FAIL(TabletMetaManager::put_del_vector(data_store, &wb, tablet_id, id, delvec));
+            ss << id << ",";
         }
         for (const auto& [rid, rowset] : _rowsets) {
             RowsetMetaPB meta_pb = rowset->rowset_meta()->to_rowset_pb();
             CHECK_FAIL(TabletMetaManager::put_rowset_meta(data_store, &wb, tablet_id, meta_pb));
         }
 
         _next_rowset_id = new_next_rowset_id;
+        ss << " next_rowset_id after:" << _next_rowset_id;
 
         _to_updates_pb_unlocked(new_tablet_meta_pb.mutable_updates());
         VLOG(2) << new_tablet_meta_pb.updates().DebugString();
@@ -3646,7 +3691,7 @@ Status TabletUpdates::load_snapshot(const SnapshotMeta& snapshot_meta, bool rest
         index_entry->value().unload();
         index_cache.release(index_entry);
 
-        LOG(INFO) << "load full snapshot done " << _debug_string(false);
+        LOG(INFO) << "load full snapshot done " << _debug_string(false) << ss.str();
 
         return Status::OK();
     } else {

diff --git a/be/src/storage/tablet_updates.h b/be/src/storage/tablet_updates.h
@@ -364,6 +364,8 @@ class TabletUpdates {
 
     std::string _debug_version_info(bool lock) const;
 
+    std::string _debug_compaction_stats(const std::vector<uint32_t>& input_rowsets, const uint32_t output_rowset);
+
     void _print_rowsets(std::vector<uint32_t>& rowsets, std::string* dst, bool abbr) const;
 
     void _set_error(const string& msg);

diff --git a/be/src/util/pretty_printer.h b/be/src/util/pretty_printer.h
@@ -176,6 +176,32 @@ class PrettyPrinter {
     /// Convenience method
     static std::string print_bytes(int64_t value) { return PrettyPrinter::print(value, TUnit::BYTES); }
 
+    // convert a vector of int to a string, consecutive numbers are printed as a range
+    // integers in input must be sorted & unique
+    template <typename T>
+    static std::string print_unique_int_list_range(const std::vector<T>& vs) {
+        std::stringstream ss;
+        if (vs.size() > 0) {
+            size_t start = 0;
+            for (size_t i = 1; i < vs.size(); ++i) {
+                if (vs[i] != vs[i - 1] + 1) {
+                    if (start == i - 1) {
+                        ss << vs[start] << ",";
+                    } else {
+                        ss << vs[start] << "-" << vs[i - 1] << ",";
+                    }
+                    start = i;
+                }
+            }
+            if (start == vs.size() - 1) {
+                ss << vs[start];
+            } else {
+                ss << vs[start] << "-" << vs[vs.size() - 1];
+            }
+        }
+        return ss.str();
+    }
+
 private:
     static const int PRECISION = 2;
     static const int TIME_NS_PRECISION = 3;