From d4527030cbe33d3e972dc70df2fae81830aa6c6e Mon Sep 17 00:00:00 2001 From: Yixuan Wang Date: Thu, 20 Aug 2026 16:49:13 +0800 Subject: [PATCH] 1 --- cloud/src/recycler/checker.cpp | 664 ++++++++++++++++++++++++++++++--- cloud/src/recycler/checker.h | 59 +++ cloud/src/recycler/recycler.h | 14 +- cloud/test/recycler_test.cpp | 299 +++++++++++++++ 4 files changed, 986 insertions(+), 50 deletions(-) diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index e122bbfb86173f..264e0c4e45d0fe 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -39,6 +40,7 @@ #include #include #include +#include #include #include "common/bvars.h" @@ -83,6 +85,90 @@ extern bool enable_inverted_check; using namespace std::chrono; +namespace { + +constexpr std::string_view kDeleteBitmapPathSuffix = "_delete_bitmap.db"; + +// Classify a V2 delete bitmap storage record by the fields written by its producers. NOT_FOUND +// means that the record does not contain the required storage discriminator; callers must report +// it as an inconsistency rather than treating it as a valid storage type. +// +// DeleteBitmapStoragePB +// | +// +-- !has_store_in_fdb -------------------------------> log warning, NOT_FOUND +// | +// +-- store_in_fdb == true ----------------------------> IN_FDB +// | +// `-- store_in_fdb == false +// +-- no packed location or empty path -----------> STANDALONE_FILE +// `-- non-empty packed_file_path -----------------> PACKED_FILE +DeleteBitmapStorageType classify_delete_bitmap_storage(const DeleteBitmapStoragePB& storage) { + if (!storage.has_store_in_fdb()) { + LOG(WARNING) << "invalid V2 delete bitmap storage: store_in_fdb is not set"; + return DeleteBitmapStorageType::NOT_FOUND; + } + + if (storage.store_in_fdb()) { + return DeleteBitmapStorageType::IN_FDB; + } + + if (!storage.has_packed_slice_location() || + storage.packed_slice_location().packed_file_path().empty()) { + return DeleteBitmapStorageType::STANDALONE_FILE; + } + return DeleteBitmapStorageType::PACKED_FILE; +} + +void cache_tablet_resource_id(const doris::RowsetMetaCloudPB& rowset, std::string* resource_id) { + if (rowset.resource_id().empty()) { + return; + } + if (resource_id->empty()) { + *resource_id = rowset.resource_id(); + return; + } + DCHECK_EQ(*resource_id, rowset.resource_id()); +} + +bool same_slice_location(const PackedSliceLocationPB& expected, const PackedSlicePB& actual) { + return expected.has_offset() && expected.has_size() && actual.has_offset() && + actual.has_size() && expected.offset() == actual.offset() && + expected.size() == actual.size(); +} + +std::optional> parse_delete_bitmap_path(std::string_view path) { + constexpr std::string_view prefix = "data/"; + if (!path.starts_with(prefix) || !path.ends_with(kDeleteBitmapPathSuffix)) { + return std::nullopt; + } + + const std::string_view relative_path = path.substr(prefix.size()); + const size_t separator = relative_path.find('/'); + if (separator == std::string_view::npos || separator == 0 || + relative_path.find('/', separator + 1) != std::string_view::npos) { + return std::nullopt; + } + + int64_t tablet_id = 0; + const std::string_view tablet_part = relative_path.substr(0, separator); + auto [tablet_end_ptr, tablet_err] = + std::from_chars(tablet_part.data(), tablet_part.data() + tablet_part.size(), tablet_id); + if (tablet_err != std::errc {} || tablet_end_ptr != tablet_part.data() + tablet_part.size() || + tablet_id <= 0) { + return std::nullopt; + } + + const std::string_view rowset_part = relative_path.substr(separator + 1); + std::string rowset_id = + std::string(rowset_part.substr(0, rowset_part.size() - kDeleteBitmapPathSuffix.size())); + if (rowset_id.empty()) { + return std::nullopt; + } + return std::pair {tablet_id, std::move(rowset_id)}; +} + +} // namespace + TxnErrorCode collect_pending_table_stream_drops( const std::shared_ptr& txn_kv, std::string_view instance_id, std::unordered_map* pending_drops) { @@ -266,6 +352,12 @@ int Checker::start() { if (int ret = checker->do_delete_bitmap_inverted_check(); ret != 0) { success = false; } + + // Check V2 delete bitmap (DBM) metadata and object storage independently from V1. + log_progress("do_delete_bitmap_storage_v2_check"); + if (int ret = checker->do_delete_bitmap_storage_v2_check(); ret != 0) { + success = false; + } } if (config::enable_mow_job_key_check) { @@ -1213,7 +1305,6 @@ int InstanceChecker::collect_tablet_rowsets( auto begin = meta_rowset_key({instance_id_, tablet_id, 0}); auto end = meta_rowset_key({instance_id_, tablet_id + 1, 0}); - int64_t rowsets_num {0}; while (it == nullptr /* may be not init */ || (it->more() && !stopped())) { TxnErrorCode err = txn->get(begin, end, &it); if (err != TxnErrorCode::TXN_OK) { @@ -1231,7 +1322,6 @@ int InstanceChecker::collect_tablet_rowsets( return -1; } - ++rowsets_num; collect_cb(rowset); if (!it->has_next()) { @@ -1242,10 +1332,6 @@ int InstanceChecker::collect_tablet_rowsets( } } - LOG(INFO) << fmt::format( - "[delete bitmap checker] successfully collect rowsets for instance_id={}, " - "tablet_id={}, rowsets_num={}", - instance_id_, tablet_id, rowsets_num); return 0; } @@ -1556,6 +1642,332 @@ int InstanceChecker::get_pending_delete_bitmap_keys( return 0; } +int InstanceChecker::scan_delete_bitmap_storage_v2( + const std::function& + callback) { + const std::string begin = versioned::meta_delete_bitmap_key({instance_id_, 0, ""}); + const std::string end = versioned::meta_delete_bitmap_key( + {instance_id_, std::numeric_limits::max(), ""}); + auto it = blob_get_range(txn_kv_, begin, end); + int ret = 0; + while (it->valid() && !stopped()) { + std::string_view key = it->key(); + if (key.empty()) { + LOG(WARNING) << "empty versioned delete bitmap key"; + return -1; + } + + key.remove_prefix(1); + std::vector, int, int>> decoded; + if (decode_key(&key, &decoded) != 0) { + LOG(WARNING) << "failed to decode versioned delete bitmap key, key=" << hex(it->key()); + return -1; + } + + const auto tablet_id = std::get(std::get<0>(decoded[3])); + const auto rowset_id = std::get(std::get<0>(decoded[4])); + if (tablet_id <= 0 || rowset_id.empty()) { + LOG(WARNING) << "invalid versioned delete bitmap key, key=" << hex(it->key()); + return -1; + } + DeleteBitmapStoragePB storage; + if (!it->parse_value(&storage)) { + LOG(WARNING) << "failed to parse versioned delete bitmap storage, key=" + << hex(it->key()); + return -1; + } + + int callback_ret = callback(tablet_id, rowset_id, storage); + TEST_SYNC_POINT_CALLBACK("InstanceChecker::scan_delete_bitmap_storage_v2.callback", + &tablet_id, &rowset_id, &callback_ret); + if (callback_ret < 0 && ret >= 0) { + ret = callback_ret; + } else if (ret >= 0) { + ret = std::max(ret, callback_ret); + } + it->next(); + } + + if (it->error_code() != TxnErrorCode::TXN_OK) { + LOG(WARNING) << "failed to scan versioned delete bitmap storage, err=" << it->error_code(); + return -1; + } + return ret; +} + +int InstanceChecker::find_rowset(int64_t tablet_id, std::string_view rowset_id, + TabletRowsetCache* rowset_cache) { + if (rowset_cache->tablet_id != tablet_id) { + rowset_cache->tablet_id = tablet_id; + rowset_cache->active_rowset_ids.clear(); + rowset_cache->resource_id.clear(); + int ret = collect_tablet_rowsets(tablet_id, [&](const doris::RowsetMetaCloudPB& rowset) { + rowset_cache->active_rowset_ids.insert(rowset.rowset_id_v2()); + cache_tablet_resource_id(rowset, &rowset_cache->resource_id); + }); + if (ret != 0) { + return ret; + } + } + + if (rowset_cache->active_rowset_ids.contains(std::string(rowset_id))) { + return 0; + } + return 1; +} + +int InstanceChecker::get_delete_bitmap_storage(int64_t tablet_id, std::string_view rowset_id, + DeleteBitmapStoragePB* storage) { + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + return -1; + } + + ValueBuf value; + const std::string key = + versioned::meta_delete_bitmap_key({instance_id_, tablet_id, std::string(rowset_id)}); + err = blob_get(txn.get(), key, &value); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + return 1; + } + if (err != TxnErrorCode::TXN_OK) { + return -1; + } + if (!value.to_pb(storage)) { + LOG(WARNING) << "failed to parse V2 delete bitmap storage, key=" << hex(key); + return -1; + } + return 0; +} + +int InstanceChecker::check_standalone_delete_bitmap_object(int64_t tablet_id, + std::string_view rowset_id, + std::string_view resource_id) { + auto* accessor = get_accessor(std::string(resource_id)); + if (accessor == nullptr) { + LOG(WARNING) << "resource not found for standalone V2 delete bitmap, resource_id=" + << resource_id; + return -1; + } + + const std::string path = delete_bitmap_path(tablet_id, std::string(rowset_id)); + int exists = accessor->exists(path); + if (exists < 0) { + return -1; + } + if (exists != 0) { + LOG(WARNING) << "standalone V2 delete bitmap object not found, path=" << path; + return 1; + } + return 0; +} + +int InstanceChecker::check_packed_delete_bitmap_object(const DeleteBitmapStoragePB& storage, + std::string_view resource_id) { + const std::string& packed_file_path = storage.packed_slice_location().packed_file_path(); + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + return -1; + } + + std::string value; + err = txn->get(packed_file_key({instance_id_, packed_file_path}), &value); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + LOG(WARNING) << "packed file metadata not found for V2 delete bitmap, path=" + << packed_file_path; + return 1; + } + if (err != TxnErrorCode::TXN_OK) { + return -1; + } + + PackedFileInfoPB packed_info; + if (!packed_info.ParseFromString(value)) { + LOG(WARNING) << "malformed packed file metadata, path=" << packed_file_path; + return -1; + } + if (packed_info.resource_id() != resource_id) { + LOG(WARNING) << "packed file resource mismatch for V2 delete bitmap, path=" + << packed_file_path << ", rowset_resource_id=" << resource_id + << ", packed_file_resource_id=" << packed_info.resource_id(); + return 1; + } + + return check_packed_file_object(packed_file_path, packed_info); +} + +int InstanceChecker::check_packed_file_object(const std::string& packed_file_path, + const PackedFileInfoPB& packed_info) { + auto* accessor = get_accessor(packed_info.resource_id()); + if (accessor == nullptr) { + LOG(WARNING) << "accessor not found for packed file, resource_id=" + << packed_info.resource_id() << ", packed_file_path=" << packed_file_path; + return -1; + } + + const int exists = accessor->exists(packed_file_path); + if (exists < 0) { + LOG(WARNING) << "failed to check packed file existence, packed_file_path=" + << packed_file_path << ", ret=" << exists; + return -1; + } + if (exists != 0 && + !(packed_info.ref_cnt() == 0 && packed_info.state() == PackedFileInfoPB::RECYCLING)) { + LOG(WARNING) << "packed file not found in storage but metadata is invalid, " + "packed_file_path=" + << packed_file_path << ", ref_cnt=" << packed_info.ref_cnt() + << " (expected=0), state=" << packed_info.state() + << " (expected=RECYCLING), ret=" << exists; + return 1; + } + return 0; +} + +int InstanceChecker::check_delete_bitmap_storage_entry(int64_t tablet_id, + std::string_view rowset_id, + const DeleteBitmapStoragePB& storage, + TabletRowsetCache* rowset_cache) { + auto storage_type = classify_delete_bitmap_storage(storage); + if (storage_type == DeleteBitmapStorageType::NOT_FOUND) { + LOG(WARNING) << "invalid V2 delete bitmap storage, tablet_id=" << tablet_id + << ", rowset_id=" << rowset_id; + return 1; + } + + int ret = find_rowset(tablet_id, rowset_id, rowset_cache); + if (ret < 0) { + return ret; + } + if (ret > 0) { + DeleteBitmapStoragePB current_storage; + ret = get_delete_bitmap_storage(tablet_id, rowset_id, ¤t_storage); + if (ret != 0) { + return ret < 0 ? ret : 0; + } + LOG(WARNING) << "rowset not found for V2 delete bitmap, tablet_id=" << tablet_id + << ", rowset_id=" << rowset_id; + return 1; + } + if (storage_type == DeleteBitmapStorageType::IN_FDB) { + return 0; + } + const std::string& resource_id = rowset_cache->resource_id; + if (resource_id.empty()) { + LOG(WARNING) << "resource id missing for external V2 delete bitmap, tablet_id=" << tablet_id + << ", rowset_id=" << rowset_id; + return 1; + } + if (storage_type == DeleteBitmapStorageType::STANDALONE_FILE) { + return check_standalone_delete_bitmap_object(tablet_id, rowset_id, resource_id); + } + return check_packed_delete_bitmap_object(storage, resource_id); +} + +int InstanceChecker::check_delete_bitmap_storage_from_kv() { + TabletRowsetCache rowset_cache; + return scan_delete_bitmap_storage_v2([&](int64_t tablet_id, std::string_view rowset_id, + const DeleteBitmapStoragePB& storage) { + return check_delete_bitmap_storage_entry(tablet_id, rowset_id, storage, &rowset_cache); + }); +} + +int InstanceChecker::check_delete_bitmap_storage_from_object() { + int check_result = 0; + TabletRowsetCache rowset_cache; + for (const auto& [resource_id, accessor] : accessor_map_) { + std::unique_ptr list_iter; + if (accessor->list_directory("data", &list_iter) != 0) { + return -1; + } + for (auto file = list_iter->next(); file.has_value(); file = list_iter->next()) { + if (!file->path.ends_with(kDeleteBitmapPathSuffix)) { + continue; + } + + auto parse_ret = parse_delete_bitmap_path(file->path); + if (!parse_ret.has_value()) { + LOG(WARNING) << "malformed delete bitmap object path, path=" << file->path; + check_result = 1; + continue; + } + const auto& [tablet_id, rowset_id] = *parse_ret; + + DeleteBitmapStoragePB storage; + int ret = get_delete_bitmap_storage(tablet_id, rowset_id, &storage); + if (ret < 0) { + return ret; + } + auto storage_type = ret == 0 ? classify_delete_bitmap_storage(storage) + : DeleteBitmapStorageType::NOT_FOUND; + if (storage_type != DeleteBitmapStorageType::STANDALONE_FILE) { + if (ret > 0) { + int exists = accessor->exists(file->path); + if (exists < 0) { + return -1; + } + if (exists != 0) { + continue; + } + } + LOG(WARNING) << "delete bitmap object has no standalone V2 metadata, path=" + << file->path; + check_result = 1; + continue; + } + + ret = find_rowset(tablet_id, rowset_id, &rowset_cache); + if (ret < 0) { + return ret; + } + if (ret > 0) { + DeleteBitmapStoragePB current_storage; + ret = get_delete_bitmap_storage(tablet_id, rowset_id, ¤t_storage); + if (ret > 0) { + int exists = accessor->exists(file->path); + if (exists < 0) { + return -1; + } + if (exists != 0) { + continue; + } + } else if (ret < 0) { + return ret; + } else { + ret = 1; + } + } + if (ret > 0 || rowset_cache.resource_id != resource_id) { + LOG(WARNING) << "delete bitmap object rowset or resource mismatch, path=" + << file->path << ", object_resource_id=" << resource_id + << ", tablet_resource_id=" << rowset_cache.resource_id; + check_result = 1; + } + } + if (!list_iter->is_valid()) { + return -1; + } + } + return check_result; +} + +int InstanceChecker::do_delete_bitmap_storage_v2_check() { + int kv_check_result = check_delete_bitmap_storage_from_kv(); + if (kv_check_result < 0) { + return kv_check_result; + } + int object_check_result = check_delete_bitmap_storage_from_object(); + if (object_check_result < 0) { + return object_check_result; + } + + int check_result = std::max(kv_check_result, object_check_result); + LOG(INFO) << "finish V2 delete bitmap storage check, instance_id=" << instance_id_ + << ", check_result=" << check_result; + return check_result; +} + int InstanceChecker::check_inverted_index_file_storage_format_v1( int64_t tablet_id, const std::string& file_path, const std::string& rowset_info, RowsetIndexesFormatV1& rowset_index_cache_v1) { @@ -1928,12 +2340,6 @@ int InstanceChecker::check_delete_bitmap_storage_optimize_v2( if (!failed_versions.empty()) { print_failed_versions(); } - LOG(INFO) << fmt::format( - "[delete bitmap checker] finish check delete bitmap storage optimize v2 for " - "instance_id={}, tablet_id={}, rowsets_num={}, " - "rowsets_with_useless_delete_bitmap_version={}", - instance_id_, tablet_id, tablet_rowsets_map.size(), - rowsets_with_useless_delete_bitmap_version); return (rowsets_with_useless_delete_bitmap_version > 1 ? 1 : 0); } @@ -3239,6 +3645,161 @@ void InstanceChecker::get_all_accessor(std::vector* acces } } +int InstanceChecker::check_delete_bitmap_packed_slice_location( + std::string_view resource_id, std::string_view small_file_path, + const PackedSliceLocationPB& location, long* num_small_file_ref_mismatch) { + std::unique_ptr txn; + TxnErrorCode err = txn_kv_->create_txn(&txn); + if (err != TxnErrorCode::TXN_OK) { + return -1; + } + std::string packed_value; + const std::string packed_key = packed_file_key({instance_id_, location.packed_file_path()}); + err = txn->get(packed_key, &packed_value); + if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) { + LOG(WARNING) << "packed file metadata not found, packed_file_path=" + << location.packed_file_path() << ", small_file_path=" << small_file_path; + ++*num_small_file_ref_mismatch; + return 1; + } + if (err != TxnErrorCode::TXN_OK) { + return -1; + } + + PackedFileInfoPB packed_info; + if (!packed_info.ParseFromString(packed_value)) { + LOG(WARNING) << "malformed packed file metadata, packed_file_path=" + << location.packed_file_path(); + return -1; + } + if (packed_info.resource_id() != resource_id) { + LOG(WARNING) << "packed file resource mismatch, packed_file_path=" + << location.packed_file_path() << ", expected_resource_id=" << resource_id + << ", actual_resource_id=" << packed_info.resource_id(); + ++*num_small_file_ref_mismatch; + return 1; + } + + int matched_count = 0; + bool location_matches = false; + for (const auto& slice : packed_info.slices()) { + if (!slice.deleted() && slice.path() == small_file_path) { + ++matched_count; + location_matches = same_slice_location(location, slice); + } + } + if (matched_count != 1 || !location_matches) { + LOG(WARNING) << "packed slice reference mismatch, packed_file_path=" + << location.packed_file_path() << ", small_file_path=" << small_file_path + << ", matched_count=" << matched_count + << ", location_matches=" << location_matches; + ++*num_small_file_ref_mismatch; + return 1; + } + return 0; +} + +int InstanceChecker::check_delete_bitmap_packed_reference(int64_t tablet_id, + std::string_view rowset_id, + const DeleteBitmapStoragePB& storage, + TabletRowsetCache* rowset_cache, + long* num_small_file_ref_mismatch) { + auto storage_type = classify_delete_bitmap_storage(storage); + if (storage_type == DeleteBitmapStorageType::NOT_FOUND) { + ++*num_small_file_ref_mismatch; + return 1; + } + if (storage_type != DeleteBitmapStorageType::PACKED_FILE) { + return 0; + } + + int ret = find_rowset(tablet_id, rowset_id, rowset_cache); + if (ret < 0) { + return ret; + } + if (ret > 0) { + DeleteBitmapStoragePB current_storage; + ret = get_delete_bitmap_storage(tablet_id, rowset_id, ¤t_storage); + if (ret > 0) { + return 0; + } + if (ret < 0) { + return ret; + } + ret = 1; + } + if (ret > 0 || rowset_cache->resource_id.empty()) { + LOG(WARNING) << "tablet resource missing for packed delete bitmap, tablet_id=" << tablet_id + << ", rowset_id=" << rowset_id; + ++*num_small_file_ref_mismatch; + return 1; + } + const std::string small_file_path = delete_bitmap_path(tablet_id, std::string(rowset_id)); + return check_delete_bitmap_packed_slice_location(rowset_cache->resource_id, small_file_path, + storage.packed_slice_location(), + num_small_file_ref_mismatch); +} + +int InstanceChecker::check_delete_bitmap_slice_reference(std::string_view packed_file_path, + const PackedSlicePB& slice, + TabletRowsetCache* rowset_cache, + std::string_view packed_file_resource_id, + long* num_small_file_ref_mismatch) { + auto parse_ret = parse_delete_bitmap_path(slice.path()); + if (!parse_ret.has_value()) { + ++*num_small_file_ref_mismatch; + return 1; + } + const auto& [tablet_id, rowset_id] = *parse_ret; + if (!slice.has_tablet_id() || !slice.has_rowset_id() || slice.tablet_id() != tablet_id || + slice.rowset_id() != rowset_id) { + LOG(WARNING) << "delete bitmap slice identifier mismatch, packed_file_path=" + << packed_file_path << ", small_file_path=" << slice.path(); + ++*num_small_file_ref_mismatch; + return 1; + } + + DeleteBitmapStoragePB storage; + int ret = get_delete_bitmap_storage(tablet_id, rowset_id, &storage); + if (ret < 0) { + return ret; + } + auto storage_type = + ret == 0 ? classify_delete_bitmap_storage(storage) : DeleteBitmapStorageType::NOT_FOUND; + if (storage_type != DeleteBitmapStorageType::PACKED_FILE) { + LOG(WARNING) << "delete bitmap slice has no packed V2 metadata, packed_file_path=" + << packed_file_path << ", small_file_path=" << slice.path(); + ++*num_small_file_ref_mismatch; + return 1; + } + + ret = find_rowset(tablet_id, rowset_id, rowset_cache); + if (ret < 0) { + return ret; + } + if (ret > 0) { + DeleteBitmapStoragePB current_storage; + ret = get_delete_bitmap_storage(tablet_id, rowset_id, ¤t_storage); + if (ret > 0) { + return 0; + } + if (ret < 0) { + return ret; + } + ret = 1; + } + const auto& location = storage.packed_slice_location(); + if (ret > 0 || rowset_cache->resource_id.empty() || + rowset_cache->resource_id != packed_file_resource_id || + location.packed_file_path() != packed_file_path || !same_slice_location(location, slice)) { + LOG(WARNING) << "delete bitmap slice metadata mismatch, packed_file_path=" + << packed_file_path << ", small_file_path=" << slice.path(); + ++*num_small_file_ref_mismatch; + return 1; + } + return 0; +} + int InstanceChecker::do_packed_file_check() { LOG(INFO) << "begin to check packed files, instance_id=" << instance_id_; int check_ret = 0; @@ -3377,6 +3938,37 @@ int InstanceChecker::do_packed_file_check() { } } + TabletRowsetCache delete_bitmap_rowset_cache; + int ret = scan_delete_bitmap_storage_v2([&](int64_t tablet_id, std::string_view rowset_id, + const DeleteBitmapStoragePB&) { + DeleteBitmapStoragePB current_storage; + int get_ret = get_delete_bitmap_storage(tablet_id, rowset_id, ¤t_storage); + if (get_ret != 0) { + return get_ret < 0 ? get_ret : 0; + } + auto storage_type = classify_delete_bitmap_storage(current_storage); + if (storage_type == DeleteBitmapStorageType::NOT_FOUND) { + ++num_small_file_ref_mismatch; + return 1; + } + if (storage_type != DeleteBitmapStorageType::PACKED_FILE) { + return 0; + } + const auto& location = current_storage.packed_slice_location(); + const std::string small_file_path = delete_bitmap_path(tablet_id, std::string(rowset_id)); + ++expected_ref_counts[location.packed_file_path()]; + packed_file_small_files[location.packed_file_path()].insert(small_file_path); + return check_delete_bitmap_packed_reference(tablet_id, rowset_id, current_storage, + &delete_bitmap_rowset_cache, + &num_small_file_ref_mismatch); + }); + if (ret < 0) { + return ret; + } + if (ret > 0) { + check_ret = 1; + } + // Step 2: Scan all packed file metadata and verify // Also collect all packed file paths from metadata for Step 3 // Map: resource_id -> set of packed_file_paths @@ -3428,45 +4020,21 @@ int InstanceChecker::do_packed_file_check() { continue; } - // Step 2.1: Verify packed file exists in storage + // Step 2.1: Verify packed file object and recycling state if (!packed_info.resource_id().empty()) { // Collect packed file path for Step 3 packed_files_in_metadata[packed_info.resource_id()].insert(packed_file_path); - auto* accessor = get_accessor(packed_info.resource_id()); - if (accessor == nullptr) { - LOG(WARNING) << "accessor not found for packed file, resource_id=" - << packed_info.resource_id() - << ", packed_file_path=" << packed_file_path; - check_ret = -1; - continue; - } - - int ret = accessor->exists(packed_file_path); - if (ret < 0) { - LOG(WARNING) << "failed to check packed file existence, packed_file_path=" - << packed_file_path << ", ret=" << ret; + int object_check_ret = check_packed_file_object(packed_file_path, packed_info); + if (object_check_ret < 0) { check_ret = -1; continue; } - - if (ret != 0) { - // ret == 1 means file not found, ret > 1 means other error - // When packed file doesn't exist in storage, ref_cnt must be 0 and state must be RECYCLING - bool ref_cnt_valid = (packed_info.ref_cnt() == 0); - bool state_valid = (packed_info.state() == cloud::PackedFileInfoPB::RECYCLING); - if (!ref_cnt_valid || !state_valid) { - LOG(WARNING) << "packed file not found in storage but metadata is invalid, " - "packed_file_path=" - << packed_file_path << ", ref_cnt=" << packed_info.ref_cnt() - << " (expected=0), state=" << packed_info.state() - << " (expected=RECYCLING), ret=" << ret; - num_packed_file_loss++; - check_ret = 1; // Data inconsistency identified - } - // If ref_cnt == 0 and state == RECYCLING, this is expected (file is being recycled) + if (object_check_ret > 0) { + num_packed_file_loss++; + check_ret = 1; // Data inconsistency identified } - // ret == 0 means file exists, which is expected + // A zero result means the object exists or is in the expected recycling state. } // Step 2.2: Verify reference count matches expected count @@ -3484,6 +4052,16 @@ int InstanceChecker::do_packed_file_check() { for (const auto& small_file : packed_info.slices()) { if (!small_file.deleted()) { small_files_in_meta.insert(small_file.path()); + if (small_file.path().ends_with(kDeleteBitmapPathSuffix)) { + int delete_bitmap_ret = check_delete_bitmap_slice_reference( + packed_file_path, small_file, &delete_bitmap_rowset_cache, + packed_info.resource_id(), &num_small_file_ref_mismatch); + if (delete_bitmap_ret < 0) { + check_ret = -1; + } else if (delete_bitmap_ret > 0) { + check_ret = 1; + } + } } } diff --git a/cloud/src/recycler/checker.h b/cloud/src/recycler/checker.h index 9a643f2f382fe9..da9aa3274487ab 100644 --- a/cloud/src/recycler/checker.h +++ b/cloud/src/recycler/checker.h @@ -39,6 +39,7 @@ namespace doris { class RowsetMetaCloudPB; +class PackedSliceLocationPB; } // namespace doris namespace doris::cloud { @@ -46,6 +47,9 @@ class StorageVaultAccessor; class InstanceChecker; class TxnKv; class InstanceInfoPB; +class DeleteBitmapStoragePB; +class PackedFileInfoPB; +class PackedSlicePB; struct PendingTableStreamDrop { int64_t base_db_id; @@ -119,6 +123,13 @@ class InstanceChecker { // Return negative if a temporary error occurred during the check process. int do_delete_bitmap_inverted_check(); + // Check consistency among V2 delete bitmap (DBM) storage metadata, active rowsets, + // standalone objects, and packed slices. + // Return 0 if success. + // Return 1 if an inconsistency is identified. + // Return negative if a temporary error occurred during the check process. + int do_delete_bitmap_storage_v2_check(); + // version = 1 : https://github.com/apache/doris/pull/40204 // checks if https://github.com/apache/doris/pull/40204 works as expected // the stale delete bitmap will be cleared in MS when BE delete expired stale rowsets @@ -185,6 +196,13 @@ class InstanceChecker { bool stopped() const { return stopped_.load(std::memory_order_acquire); } private: + struct TabletRowsetCache { + int64_t tablet_id {-1}; + std::unordered_set active_rowset_ids; + // All rowsets in a tablet belong to the same storage vault. + std::string resource_id; + }; + struct RowsetIndexesFormatV1 { std::string rowset_id; std::unordered_set segment_ids; @@ -214,9 +232,50 @@ class InstanceChecker { std::unordered_map>& tmp_rowsets); int get_pending_delete_bitmap_keys(int64_t tablet_id, std::unordered_set& pending_delete_bitmaps); + int scan_delete_bitmap_storage_v2( + const std::function& + callback); + + int check_delete_bitmap_storage_from_kv(); + + int check_delete_bitmap_storage_from_object(); + + int check_delete_bitmap_storage_entry(int64_t tablet_id, std::string_view rowset_id, + const DeleteBitmapStoragePB& storage, + TabletRowsetCache* rowset_cache); + int check_standalone_delete_bitmap_object(int64_t tablet_id, std::string_view rowset_id, + std::string_view resource_id); + + int check_packed_delete_bitmap_object(const DeleteBitmapStoragePB& storage, + std::string_view resource_id); + + int check_packed_file_object(const std::string& packed_file_path, + const PackedFileInfoPB& packed_info); + + int get_delete_bitmap_storage(int64_t tablet_id, std::string_view rowset_id, + DeleteBitmapStoragePB* storage); + + int find_rowset(int64_t tablet_id, std::string_view rowset_id, TabletRowsetCache* rowset_cache); + int check_delete_bitmap_storage_optimize_v2(int64_t tablet_id, bool has_sequence_col, int64_t& abnormal_rowsets_num); + int check_delete_bitmap_packed_slice_location(std::string_view resource_id, + std::string_view small_file_path, + const PackedSliceLocationPB& location, + long* num_small_file_ref_mismatch); + + int check_delete_bitmap_packed_reference(int64_t tablet_id, std::string_view rowset_id, + const DeleteBitmapStoragePB& storage, + TabletRowsetCache* rowset_cache, + long* num_small_file_ref_mismatch); + + int check_delete_bitmap_slice_reference(std::string_view packed_file_path, + const PackedSlicePB& slice, + TabletRowsetCache* rowset_cache, + std::string_view packed_file_resource_id, + long* num_small_file_ref_mismatch); + int check_inverted_index_file_storage_format_v1(int64_t tablet_id, const std::string& file_path, const std::string& rowset_info, RowsetIndexesFormatV1& rowset_index_cache_v1); diff --git a/cloud/src/recycler/recycler.h b/cloud/src/recycler/recycler.h index 0a3ab707ede784..8de16a3f6f9dc0 100644 --- a/cloud/src/recycler/recycler.h +++ b/cloud/src/recycler/recycler.h @@ -149,6 +149,13 @@ struct RowsetDeleteTask { std::string rowset_ref_count_key; }; +enum class DeleteBitmapStorageType { + NOT_FOUND, + IN_FDB, + STANDALONE_FILE, + PACKED_FILE, +}; + class RecyclerMetricsContext { public: RecyclerMetricsContext() = default; @@ -522,13 +529,6 @@ class InstanceRecycler { // Returns 0 for success, -1 for error. int decrement_packed_file_ref_counts(const doris::RowsetMetaCloudPB& rs_meta_pb); - enum class DeleteBitmapStorageType { - NOT_FOUND, - IN_FDB, - STANDALONE_FILE, - PACKED_FILE, - }; - // Process delete bitmap storage and decrement packed file ref count when needed. // Returns 0 for success, -1 for error. // out_storage_type: if not null, will be set to the delete bitmap storage type. diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp index 360570dd64d855..abde7b724067c9 100644 --- a/cloud/test/recycler_test.cpp +++ b/cloud/test/recycler_test.cpp @@ -38,6 +38,7 @@ #include #include #include +#include #include "common/bvars.h" #include "common/config.h" @@ -258,6 +259,98 @@ static int create_delete_bitmaps_v2(TxnKv* txn_kv, StorageVaultAccessor* accesso return accessor->put_file(delete_bitmap_path(tablet_id, rowset_id), ""); } +static int put_delete_bitmap_storage_v2(TxnKv* txn_kv, int64_t tablet_id, + const std::string& rowset_id, + const DeleteBitmapStoragePB& storage) { + std::unique_ptr txn; + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + + auto key = versioned::meta_delete_bitmap_key({instance_id, tablet_id, rowset_id}); + cloud::blob_put(txn.get(), key, storage, 0); + return txn->commit() == TxnErrorCode::TXN_OK ? 0 : -1; +} + +static int put_packed_file_info(TxnKv* txn_kv, const std::string& packed_file_path, + const PackedFileInfoPB& packed_info) { + std::unique_ptr txn; + if (txn_kv->create_txn(&txn) != TxnErrorCode::TXN_OK) { + return -1; + } + + auto key = packed_file_key({instance_id, packed_file_path}); + txn->put(key, packed_info.SerializeAsString()); + return txn->commit() == TxnErrorCode::TXN_OK ? 0 : -1; +} + +static int create_committed_rowset_with_rowset_id(TxnKv* txn_kv, StorageVaultAccessor* accessor, + const std::string& resource_id, int64_t tablet_id, + int64_t start_version, int64_t end_version, + std::string rowset_id, bool segments_overlap, + int num_segments, int64_t create_time); + +static std::pair run_packed_delete_bitmap_check( + bool write_v2_metadata, bool write_slice, int64_t actual_offset, int64_t ref_cnt, + bool write_packed_object = true, + PackedFileInfoPB::PackedFileState state = PackedFileInfoPB::NORMAL) { + auto txn_kv = std::make_shared(); + EXPECT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + instance.add_obj_info()->set_id("1"); + + InstanceChecker checker(txn_kv, instance_id); + EXPECT_EQ(checker.init(instance), 0); + auto accessor = checker.accessor_map_.begin()->second; + + constexpr int64_t tablet_id = 910005; + constexpr std::string_view rowset_id = "packed_dbm_rowset"; + constexpr std::string_view packed_file_path = "data/packed_file/dbm-packed.bin"; + constexpr int64_t slice_size = 16; + + if (write_v2_metadata) { + EXPECT_EQ(0, create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", + tablet_id, 1, 1, std::string(rowset_id), + false, 1, current_time)); + DeleteBitmapStoragePB storage; + storage.set_store_in_fdb(false); + auto* location = storage.mutable_packed_slice_location(); + location->set_packed_file_path(std::string(packed_file_path)); + location->set_offset(0); + location->set_size(slice_size); + location->set_packed_file_size(slice_size); + EXPECT_EQ(0, put_delete_bitmap_storage_v2(txn_kv.get(), tablet_id, std::string(rowset_id), + storage)); + } + + PackedFileInfoPB packed_info; + packed_info.set_ref_cnt(ref_cnt); + packed_info.set_total_slice_num(write_slice ? 1 : 0); + packed_info.set_total_slice_bytes(write_slice ? slice_size : 0); + packed_info.set_remaining_slice_bytes(write_slice ? slice_size : 0); + packed_info.set_state(state); + packed_info.set_resource_id("1"); + if (write_slice) { + auto* slice = packed_info.add_slices(); + const std::string slice_rowset_id = + write_v2_metadata ? std::string(rowset_id) : "orphan_dbm_rowset"; + slice->set_path(delete_bitmap_path(tablet_id, slice_rowset_id)); + slice->set_offset(actual_offset); + slice->set_size(slice_size); + slice->set_deleted(false); + slice->set_tablet_id(tablet_id); + slice->set_rowset_id(write_v2_metadata ? std::string(rowset_id) : "orphan_dbm_rowset"); + } + EXPECT_EQ(0, put_packed_file_info(txn_kv.get(), std::string(packed_file_path), packed_info)); + if (write_packed_object) { + EXPECT_EQ(0, accessor->put_file(std::string(packed_file_path), "packed")); + } + + return {checker.do_delete_bitmap_storage_v2_check(), checker.do_packed_file_check()}; +} + static int create_recycle_rowset(TxnKv* txn_kv, StorageVaultAccessor* accessor, const doris::RowsetMetaCloudPB& rowset, RecycleRowsetPB::Type type, bool write_schema_kv, bool enable_create_delete_bitmaps_v2 = false, @@ -6650,6 +6743,212 @@ TEST(CheckerTest, delete_bitmap_inverted_check_abnormal) { ASSERT_EQ(expected_abnormal_delete_bitmaps, real_abnormal_delete_bitmaps); } +TEST(CheckerTest, delete_bitmap_storage_v2_check_normal) { + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + auto* obj_info = instance.add_obj_info(); + obj_info->set_id("1"); + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + auto accessor = checker.accessor_map_.begin()->second; + + constexpr int64_t fdb_tablet_id = 910001; + constexpr int64_t file_tablet_id = 910002; + ASSERT_EQ(0, + create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", + fdb_tablet_id, 1, 1, "fdb_rowset", false, 1)); + ASSERT_EQ(0, create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", + fdb_tablet_id, 2, 2, "source_rowset", false, + 1)); + ASSERT_EQ(0, create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", + file_tablet_id, 1, 1, "file_rowset", false, + 1)); + + DeleteBitmapStoragePB fdb_storage; + fdb_storage.set_store_in_fdb(true); + auto* delete_bitmap = fdb_storage.mutable_delete_bitmap(); + delete_bitmap->add_rowset_ids("source_rowset"); + delete_bitmap->add_segment_ids(0); + delete_bitmap->add_versions(1); + delete_bitmap->add_segment_delete_bitmaps("bitmap"); + delete_bitmap->add_rowset_ids("source_rowset"); + delete_bitmap->add_segment_ids(0); + delete_bitmap->add_versions(2); + delete_bitmap->add_segment_delete_bitmaps("bitmap2"); + ASSERT_EQ(0, + put_delete_bitmap_storage_v2(txn_kv.get(), fdb_tablet_id, "fdb_rowset", fdb_storage)); + + DeleteBitmapStoragePB file_storage; + file_storage.set_store_in_fdb(false); + ASSERT_EQ(0, put_delete_bitmap_storage_v2(txn_kv.get(), file_tablet_id, "file_rowset", + file_storage)); + ASSERT_EQ(0, accessor->put_file(delete_bitmap_path(file_tablet_id, "file_rowset"), "bitmap")); + + ASSERT_EQ(0, checker.do_delete_bitmap_storage_v2_check()); +} + +TEST(CheckerTest, delete_bitmap_storage_v2_check_missing_standalone_object) { + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + instance.add_obj_info()->set_id("1"); + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + auto accessor = checker.accessor_map_.begin()->second; + + constexpr int64_t tablet_id = 910003; + constexpr std::string_view rowset_id = "missing_file_rowset"; + ASSERT_EQ(0, + create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id, + 1, 1, std::string(rowset_id), false, 1)); + DeleteBitmapStoragePB storage; + storage.set_store_in_fdb(false); + ASSERT_EQ(0, put_delete_bitmap_storage_v2(txn_kv.get(), tablet_id, std::string(rowset_id), + storage)); + + ASSERT_EQ(1, checker.do_delete_bitmap_storage_v2_check()); +} + +TEST(CheckerTest, delete_bitmap_storage_v2_check_missing_store_flag) { + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + instance.add_obj_info()->set_id("1"); + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + auto accessor = checker.accessor_map_.begin()->second; + + constexpr int64_t tablet_id = 910006; + constexpr std::string_view rowset_id = "missing_store_flag_rowset"; + ASSERT_EQ(0, + create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", tablet_id, + 1, 1, std::string(rowset_id), false, 1)); + DeleteBitmapStoragePB storage; + auto* location = storage.mutable_packed_slice_location(); + location->set_packed_file_path("data/packed_file/missing_store_flag.bin"); + location->set_offset(0); + location->set_size(1); + ASSERT_EQ(0, put_delete_bitmap_storage_v2(txn_kv.get(), tablet_id, std::string(rowset_id), + storage)); + + ASSERT_EQ(1, checker.do_delete_bitmap_storage_v2_check()); +} + +TEST(CheckerTest, delete_bitmap_storage_v2_check_continues_after_callback_failure) { + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + instance.add_obj_info()->set_id("1"); + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + auto accessor = checker.accessor_map_.begin()->second; + + constexpr int64_t failed_tablet_id = 910007; + constexpr int64_t succeeding_tablet_id = 910008; + ASSERT_EQ(0, create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), + "missing_resource", failed_tablet_id, 1, 1, + "failed_rowset", false, 1)); + ASSERT_EQ(0, create_committed_rowset_with_rowset_id(txn_kv.get(), accessor.get(), "1", + succeeding_tablet_id, 1, 1, + "succeeding_rowset", false, 1)); + + DeleteBitmapStoragePB failed_storage; + failed_storage.set_store_in_fdb(false); + ASSERT_EQ(0, put_delete_bitmap_storage_v2(txn_kv.get(), failed_tablet_id, "failed_rowset", + failed_storage)); + DeleteBitmapStoragePB succeeding_storage; + succeeding_storage.set_store_in_fdb(true); + ASSERT_EQ(0, put_delete_bitmap_storage_v2(txn_kv.get(), succeeding_tablet_id, + "succeeding_rowset", succeeding_storage)); + + std::vector> callback_results; + auto sp = SyncPoint::get_instance(); + SyncPoint::CallbackGuard guard; + sp->set_call_back( + "InstanceChecker::scan_delete_bitmap_storage_v2.callback", + [&callback_results](auto&& args) { + callback_results.emplace_back(*try_any_cast(args[0]), + *try_any_cast(args[2])); + }, + &guard); + sp->enable_processing(); + DORIS_CLOUD_DEFER { + SyncPoint::get_instance()->disable_processing(); + }; + + ASSERT_EQ(-1, checker.do_delete_bitmap_storage_v2_check()); + EXPECT_EQ((std::vector> {{failed_tablet_id, -1}, + {succeeding_tablet_id, 0}}), + callback_results); +} + +TEST(CheckerTest, delete_bitmap_storage_v2_check_orphan_standalone_object) { + auto txn_kv = std::make_shared(); + ASSERT_EQ(txn_kv->init(), 0); + + InstanceInfoPB instance; + instance.set_instance_id(instance_id); + auto* obj_info = instance.add_obj_info(); + obj_info->set_id("1"); + + InstanceChecker checker(txn_kv, instance_id); + ASSERT_EQ(checker.init(instance), 0); + auto accessor = checker.accessor_map_.begin()->second; + constexpr int64_t tablet_id = 910004; + ASSERT_EQ(0, accessor->put_file(delete_bitmap_path(tablet_id, "orphan_rowset"), "bitmap")); + + ASSERT_EQ(1, checker.do_delete_bitmap_storage_v2_check()); +} + +TEST(CheckerTest, packed_file_check_delete_bitmap_v2_normal) { + EXPECT_EQ((std::pair {0, 0}), run_packed_delete_bitmap_check(true, true, 0, 1)); +} + +TEST(CheckerTest, packed_file_check_delete_bitmap_v2_missing_slice) { + EXPECT_EQ((std::pair {0, 1}), run_packed_delete_bitmap_check(true, false, 0, 1)); +} + +TEST(CheckerTest, packed_file_check_delete_bitmap_v2_location_mismatch) { + EXPECT_EQ((std::pair {0, 1}), run_packed_delete_bitmap_check(true, true, 8, 1)); +} + +TEST(CheckerTest, packed_file_check_delete_bitmap_v2_ref_count_mismatch) { + EXPECT_EQ((std::pair {0, 1}), run_packed_delete_bitmap_check(true, true, 0, 0)); +} + +TEST(CheckerTest, delete_bitmap_storage_v2_check_missing_packed_object) { + EXPECT_EQ((std::pair {1, 1}), + run_packed_delete_bitmap_check(true, true, 0, 1, false)); +} + +TEST(CheckerTest, delete_bitmap_storage_v2_check_recycling_packed_object) { + EXPECT_EQ((std::pair {0, 1}), + run_packed_delete_bitmap_check(true, true, 0, 0, false, PackedFileInfoPB::RECYCLING)); +} + +TEST(CheckerTest, packed_file_check_recycling_missing_object) { + EXPECT_EQ( + (std::pair {0, 0}), + run_packed_delete_bitmap_check(false, false, 0, 0, false, PackedFileInfoPB::RECYCLING)); +} + +TEST(CheckerTest, packed_file_check_delete_bitmap_v2_orphan_slice) { + EXPECT_EQ((std::pair {0, 1}), run_packed_delete_bitmap_check(false, true, 0, 0)); +} + TEST(CheckerTest, delete_bitmap_storage_optimize_check_normal) { auto txn_kv = std::make_shared(); ASSERT_EQ(txn_kv->init(), 0);