From cbb3b259159f90775834e73b336dbdac77c36643 Mon Sep 17 00:00:00 2001 From: Levi Tamasi Date: Tue, 22 Jun 2021 09:48:50 -0700 Subject: [PATCH] Print blob file checksums as hex (#8437) Summary: Currently, blob file checksums are incorrectly dumped as raw bytes in the `ldb manifest_dump` output (i.e. they are not printed as hex). The patch fixes this and also updates some test cases to reflect that the checksum value field in `BlobFileAddition` and `SharedBlobFileMetaData` contains the raw checksum and not a hex string. Pull Request resolved: https://github.com/facebook/rocksdb/pull/8437 Test Plan: `make check` Tested using `ldb manifest_dump` Reviewed By: akankshamahajan15 Differential Revision: D29284170 Pulled By: ltamasi fbshipit-source-id: d11cfb3435b14cd73c8a3d3eb14fa0f9fa1d2228 --- db/blob/blob_file_addition.cc | 6 ++++-- db/blob/blob_file_addition_test.cc | 12 ++++++++---- db/blob/blob_file_meta.cc | 5 ++++- db/db_test.cc | 2 +- db/obsolete_files_test.cc | 2 +- db/version_builder_test.cc | 20 +++++++++++++++----- db/version_set_test.cc | 11 ++++++----- 7 files changed, 39 insertions(+), 19 deletions(-) diff --git a/db/blob/blob_file_addition.cc b/db/blob/blob_file_addition.cc index cf3b6817c..71b1bb7fc 100644 --- a/db/blob/blob_file_addition.cc +++ b/db/blob/blob_file_addition.cc @@ -135,7 +135,8 @@ std::ostream& operator<<(std::ostream& os, << " total_blob_count: " << blob_file_addition.GetTotalBlobCount() << " total_blob_bytes: " << blob_file_addition.GetTotalBlobBytes() << " checksum_method: " << blob_file_addition.GetChecksumMethod() - << " checksum_value: " << blob_file_addition.GetChecksumValue(); + << " checksum_value: " + << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true); return os; } @@ -146,7 +147,8 @@ JSONWriter& operator<<(JSONWriter& jw, << "TotalBlobCount" << blob_file_addition.GetTotalBlobCount() << "TotalBlobBytes" << blob_file_addition.GetTotalBlobBytes() << "ChecksumMethod" << blob_file_addition.GetChecksumMethod() - << "ChecksumValue" << blob_file_addition.GetChecksumValue(); + << "ChecksumValue" + << Slice(blob_file_addition.GetChecksumValue()).ToString(/* hex */ true); return jw; } diff --git a/db/blob/blob_file_addition_test.cc b/db/blob/blob_file_addition_test.cc index 587123b0b..9d819b557 100644 --- a/db/blob/blob_file_addition_test.cc +++ b/db/blob/blob_file_addition_test.cc @@ -46,7 +46,9 @@ TEST_F(BlobFileAdditionTest, NonEmpty) { constexpr uint64_t total_blob_count = 2; constexpr uint64_t total_blob_bytes = 123456; const std::string checksum_method("SHA1"); - const std::string checksum_value("bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"); + const std::string checksum_value( + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"); BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, total_blob_bytes, checksum_method, @@ -113,7 +115,9 @@ TEST_F(BlobFileAdditionTest, DecodeErrors) { ASSERT_TRUE(std::strstr(s.getState(), "checksum value")); } - constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; PutLengthPrefixedSlice(&str, checksum_value); slice = str; @@ -150,7 +154,7 @@ TEST_F(BlobFileAdditionTest, ForwardCompatibleCustomField) { constexpr uint64_t total_blob_count = 9999; constexpr uint64_t total_blob_bytes = 100000000; const std::string checksum_method("CRC32"); - const std::string checksum_value("3d87ff57"); + const std::string checksum_value("\x3d\x87\xff\x57"); BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, total_blob_bytes, checksum_method, @@ -178,7 +182,7 @@ TEST_F(BlobFileAdditionTest, ForwardIncompatibleCustomField) { constexpr uint64_t total_blob_count = 100; constexpr uint64_t total_blob_bytes = 2000000; const std::string checksum_method("CRC32B"); - const std::string checksum_value("6dbdf23a"); + const std::string checksum_value("\x6d\xbd\xf2\x3a"); BlobFileAddition blob_file_addition(blob_file_number, total_blob_count, total_blob_bytes, checksum_method, diff --git a/db/blob/blob_file_meta.cc b/db/blob/blob_file_meta.cc index dd9845f93..799f8ef21 100644 --- a/db/blob/blob_file_meta.cc +++ b/db/blob/blob_file_meta.cc @@ -8,6 +8,8 @@ #include #include +#include "rocksdb/slice.h" + namespace ROCKSDB_NAMESPACE { std::string SharedBlobFileMetaData::DebugString() const { @@ -23,7 +25,8 @@ std::ostream& operator<<(std::ostream& os, << " total_blob_count: " << shared_meta.GetTotalBlobCount() << " total_blob_bytes: " << shared_meta.GetTotalBlobBytes() << " checksum_method: " << shared_meta.GetChecksumMethod() - << " checksum_value: " << shared_meta.GetChecksumValue(); + << " checksum_value: " + << Slice(shared_meta.GetChecksumValue()).ToString(/* hex */ true); return os; } diff --git a/db/db_test.cc b/db/db_test.cc index f0de4db02..f1932d61b 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -2364,7 +2364,7 @@ TEST_F(DBTest, GetLiveBlobFiles) { constexpr uint64_t total_blob_count = 555; constexpr uint64_t total_blob_bytes = 66666; constexpr char checksum_method[] = "CRC32"; - constexpr char checksum_value[] = "3d87ff57"; + constexpr char checksum_value[] = "\x3d\x87\xff\x57"; auto shared_meta = SharedBlobFileMetaData::Create( blob_file_number, total_blob_count, total_blob_bytes, checksum_method, diff --git a/db/obsolete_files_test.cc b/db/obsolete_files_test.cc index ee6b07639..6cff8b380 100644 --- a/db/obsolete_files_test.cc +++ b/db/obsolete_files_test.cc @@ -228,7 +228,7 @@ TEST_F(ObsoleteFilesTest, BlobFiles) { constexpr uint64_t second_total_blob_count = 100; constexpr uint64_t second_total_blob_bytes = 2000000; constexpr char second_checksum_method[] = "CRC32B"; - constexpr char second_checksum_value[] = "6dbdf23a"; + constexpr char second_checksum_value[] = "\x6d\xbd\xf2\x3a"; auto shared_meta = SharedBlobFileMetaData::Create( second_blob_file_number, second_total_blob_count, second_total_blob_bytes, diff --git a/db/version_builder_test.cc b/db/version_builder_test.cc index 26b473f0f..502b70c7b 100644 --- a/db/version_builder_test.cc +++ b/db/version_builder_test.cc @@ -659,7 +659,9 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAddition) { constexpr uint64_t total_blob_count = 5678; constexpr uint64_t total_blob_bytes = 999999; constexpr char checksum_method[] = "SHA1"; - constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, checksum_method, checksum_value); @@ -703,7 +705,9 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyInBase) { constexpr uint64_t total_blob_count = 5678; constexpr uint64_t total_blob_bytes = 999999; constexpr char checksum_method[] = "SHA1"; - constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; constexpr uint64_t garbage_blob_count = 123; constexpr uint64_t garbage_blob_bytes = 456789; @@ -744,7 +748,9 @@ TEST_F(VersionBuilderTest, ApplyBlobFileAdditionAlreadyApplied) { constexpr uint64_t total_blob_count = 5678; constexpr uint64_t total_blob_bytes = 999999; constexpr char checksum_method[] = "SHA1"; - constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, checksum_method, checksum_value); @@ -764,7 +770,9 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileInBase) { constexpr uint64_t total_blob_count = 5678; constexpr uint64_t total_blob_bytes = 999999; constexpr char checksum_method[] = "SHA1"; - constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; constexpr uint64_t garbage_blob_count = 123; constexpr uint64_t garbage_blob_bytes = 456789; @@ -841,7 +849,9 @@ TEST_F(VersionBuilderTest, ApplyBlobFileGarbageFileAdditionApplied) { constexpr uint64_t total_blob_count = 5678; constexpr uint64_t total_blob_bytes = 999999; constexpr char checksum_method[] = "SHA1"; - constexpr char checksum_value[] = "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; + constexpr char checksum_value[] = + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c\x52" + "\x5c\xbd"; addition.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, checksum_method, checksum_value); diff --git a/db/version_set_test.cc b/db/version_set_test.cc index 75919c6ed..90088100c 100644 --- a/db/version_set_test.cc +++ b/db/version_set_test.cc @@ -989,7 +989,8 @@ TEST_F(VersionSetTest, PersistBlobFileStateInNewManifest) { constexpr uint64_t total_blob_bytes = 77777777; constexpr char checksum_method[] = "SHA1"; constexpr char checksum_value[] = - "bdb7f34a59dfa1592ce7f52e99f98c570c525cbd"; + "\xbd\xb7\xf3\x4a\x59\xdf\xa1\x59\x2c\xe7\xf5\x2e\x99\xf9\x8c\x57\x0c" + "\x52\x5c\xbd"; auto shared_meta = SharedBlobFileMetaData::Create( blob_file_number, total_blob_count, total_blob_bytes, checksum_method, @@ -1010,7 +1011,7 @@ TEST_F(VersionSetTest, PersistBlobFileStateInNewManifest) { constexpr uint64_t total_blob_count = 555; constexpr uint64_t total_blob_bytes = 66666; constexpr char checksum_method[] = "CRC32"; - constexpr char checksum_value[] = "3d87ff57"; + constexpr char checksum_value[] = "\x3d\x87\xff\x57"; auto shared_meta = SharedBlobFileMetaData::Create( blob_file_number, total_blob_count, total_blob_bytes, checksum_method, @@ -1068,7 +1069,7 @@ TEST_F(VersionSetTest, AddLiveBlobFiles) { constexpr uint64_t first_total_blob_count = 555; constexpr uint64_t first_total_blob_bytes = 66666; constexpr char first_checksum_method[] = "CRC32"; - constexpr char first_checksum_value[] = "3d87ff57"; + constexpr char first_checksum_value[] = "\x3d\x87\xff\x57"; auto first_shared_meta = SharedBlobFileMetaData::Create( first_blob_file_number, first_total_blob_count, first_total_blob_bytes, @@ -1111,7 +1112,7 @@ TEST_F(VersionSetTest, AddLiveBlobFiles) { constexpr uint64_t second_total_blob_count = 100; constexpr uint64_t second_total_blob_bytes = 2000000; constexpr char second_checksum_method[] = "CRC32B"; - constexpr char second_checksum_value[] = "6dbdf23a"; + constexpr char second_checksum_value[] = "\x6d\xbd\xf2\x3a"; auto second_shared_meta = SharedBlobFileMetaData::Create( second_blob_file_number, second_total_blob_count, second_total_blob_bytes, @@ -1151,7 +1152,7 @@ TEST_F(VersionSetTest, ObsoleteBlobFile) { constexpr uint64_t total_blob_count = 555; constexpr uint64_t total_blob_bytes = 66666; constexpr char checksum_method[] = "CRC32"; - constexpr char checksum_value[] = "3d87ff57"; + constexpr char checksum_value[] = "\x3d\x87\xff\x57"; edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes, checksum_method, checksum_value);