Remove unused fields from FileMetaData (temporarily) (#10443)

Summary:
FileMetaData::[min|max]_timestamp are not currently being used or
tracked by RocksDB, even when user-defined timestamp is enabled. Each of
them is a std::string which can occupy 32 bytes. Remove them for now.
They may be added back when we have a pressing need for them. When we do
add them back, consider store them in a more compact way, e.g. one
boolean flag and a byte array of size 16.

Per file min/max timestamp bounds are available as table properties.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10443

Test Plan: make check

Reviewed By: pdillinger

Differential Revision: D38292275

Pulled By: riversand963

fbshipit-source-id: 841dc4e855ad8f8481c80cb020603de9607c9c94
main
Yanqin Jin 2 years ago committed by Facebook GitHub Bot
parent cc2099803a
commit fbfcf5cbcd
  1. 1
      HISTORY.md
  2. 3
      db/compaction/compaction_job_test.cc
  3. 3
      db/compaction/compaction_picker_test.cc
  4. 5
      db/db_impl/db_impl_compaction_flush.cc
  5. 14
      db/db_impl/db_impl_experimental.cc
  6. 3
      db/db_impl/db_impl_open.cc
  7. 15
      db/experimental.cc
  8. 3
      db/external_sst_file_ingestion_job.cc
  9. 3
      db/flush_job.cc
  10. 2
      db/import_column_family_job.cc
  11. 1
      db/repair.cc
  12. 99
      db/version_builder_test.cc
  13. 32
      db/version_edit.cc
  14. 14
      db/version_edit.h
  15. 33
      db/version_edit_test.cc
  16. 3
      db/version_set.cc
  17. 21
      db/version_set_test.cc

@ -22,6 +22,7 @@
### Behavior Change
* Added checksum handshake during the copying of decompressed WAL fragment. This together with #9875, #10037, #10212, #10114 and #10319 provides end-to-end integrity protection for write batch during recovery.
* PosixLogger is removed and by default EnvLogger will be used for info logging. The behavior of the two loggers should be very similar when using the default Posix Env.
* Remove [min|max]_timestamp from VersionEdit for now since they are not tracked in MANIFEST anyway but consume two empty std::string (up to 64 bytes) for each file. Should they be added back in the future, we should store them more compactly.
## 7.5.0 (07/15/2022)
### New Features

@ -372,8 +372,7 @@ class CompactionJobTestBase : public testing::Test {
smallest_seqno, largest_seqno, false, Temperature::kUnknown,
oldest_blob_file_number, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
mutex_.Lock();
EXPECT_OK(

@ -116,8 +116,7 @@ class CompactionPickerTest : public testing::Test {
InternalKey(largest, largest_seq, kTypeValue), smallest_seq,
largest_seq, marked_for_compact, temperature, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
f->compensated_file_size =
(compensated_file_size != 0) ? compensated_file_size : file_size;
f->oldest_ancester_time = oldest_ancestor_time;

@ -1686,8 +1686,7 @@ Status DBImpl::ReFitLevel(ColumnFamilyData* cfd, int level, int target_level) {
f->smallest, f->largest, f->fd.smallest_seqno, f->fd.largest_seqno,
f->marked_for_compaction, f->temperature, f->oldest_blob_file_number,
f->oldest_ancester_time, f->file_creation_time, f->file_checksum,
f->file_checksum_func_name, f->min_timestamp, f->max_timestamp,
f->unique_id);
f->file_checksum_func_name, f->unique_id);
}
ROCKS_LOG_DEBUG(immutable_db_options_.info_log,
"[%s] Apply version edit:\n%s", cfd->GetName().c_str(),
@ -3313,7 +3312,7 @@ Status DBImpl::BackgroundCompaction(bool* made_progress,
f->fd.largest_seqno, f->marked_for_compaction, f->temperature,
f->oldest_blob_file_number, f->oldest_ancester_time,
f->file_creation_time, f->file_checksum, f->file_checksum_func_name,
f->min_timestamp, f->max_timestamp, f->unique_id);
f->unique_id);
ROCKS_LOG_BUFFER(
log_buffer,

@ -131,13 +131,13 @@ Status DBImpl::PromoteL0(ColumnFamilyHandle* column_family, int target_level) {
edit.SetColumnFamily(cfd->GetID());
for (const auto& f : l0_files) {
edit.DeleteFile(0, f->fd.GetNumber());
edit.AddFile(
target_level, f->fd.GetNumber(), f->fd.GetPathId(),
f->fd.GetFileSize(), f->smallest, f->largest, f->fd.smallest_seqno,
f->fd.largest_seqno, f->marked_for_compaction, f->temperature,
f->oldest_blob_file_number, f->oldest_ancester_time,
f->file_creation_time, f->file_checksum, f->file_checksum_func_name,
f->min_timestamp, f->max_timestamp, f->unique_id);
edit.AddFile(target_level, f->fd.GetNumber(), f->fd.GetPathId(),
f->fd.GetFileSize(), f->smallest, f->largest,
f->fd.smallest_seqno, f->fd.largest_seqno,
f->marked_for_compaction, f->temperature,
f->oldest_blob_file_number, f->oldest_ancester_time,
f->file_creation_time, f->file_checksum,
f->file_checksum_func_name, f->unique_id);
}
status = versions_->LogAndApply(cfd, *cfd->GetLatestMutableCFOptions(),

@ -1600,8 +1600,7 @@ Status DBImpl::WriteLevel0TableForRecovery(int job_id, ColumnFamilyData* cfd,
meta.marked_for_compaction, meta.temperature,
meta.oldest_blob_file_number, meta.oldest_ancester_time,
meta.file_creation_time, meta.file_checksum,
meta.file_checksum_func_name, meta.min_timestamp,
meta.max_timestamp, meta.unique_id);
meta.file_checksum_func_name, meta.unique_id);
for (const auto& blob : blob_file_additions) {
edit->AddBlobFile(blob);

@ -105,14 +105,13 @@ Status UpdateManifestForFilesState(
// Current state inconsistent with manifest
++files_updated;
edit.DeleteFile(level, number);
edit.AddFile(level, number, lf->fd.GetPathId(),
lf->fd.GetFileSize(), lf->smallest, lf->largest,
lf->fd.smallest_seqno, lf->fd.largest_seqno,
lf->marked_for_compaction, temp,
lf->oldest_blob_file_number,
lf->oldest_ancester_time, lf->file_creation_time,
lf->file_checksum, lf->file_checksum_func_name,
lf->min_timestamp, lf->max_timestamp, lf->unique_id);
edit.AddFile(
level, number, lf->fd.GetPathId(), lf->fd.GetFileSize(),
lf->smallest, lf->largest, lf->fd.smallest_seqno,
lf->fd.largest_seqno, lf->marked_for_compaction, temp,
lf->oldest_blob_file_number, lf->oldest_ancester_time,
lf->file_creation_time, lf->file_checksum,
lf->file_checksum_func_name, lf->unique_id);
}
}
} else {

@ -450,8 +450,7 @@ Status ExternalSstFileIngestionJob::Run() {
f.smallest_internal_key, f.largest_internal_key, f.assigned_seqno,
f.assigned_seqno, false, f.file_temperature, kInvalidBlobFileNumber,
oldest_ancester_time, current_time, f.file_checksum,
f.file_checksum_func_name, kDisableUserTimestamp, kDisableUserTimestamp,
f.unique_id);
f.file_checksum_func_name, f.unique_id);
f_metadata.temperature = f.file_temperature;
edit_.AddFile(f.picked_level, f_metadata);
}

@ -1000,8 +1000,7 @@ Status FlushJob::WriteLevel0Table() {
meta_.marked_for_compaction, meta_.temperature,
meta_.oldest_blob_file_number, meta_.oldest_ancester_time,
meta_.file_creation_time, meta_.file_checksum,
meta_.file_checksum_func_name, meta_.min_timestamp,
meta_.max_timestamp, meta_.unique_id);
meta_.file_checksum_func_name, meta_.unique_id);
edit_->SetBlobFileAdditions(std::move(blob_file_additions));
}

@ -160,7 +160,7 @@ Status ImportColumnFamilyJob::Run() {
file_metadata.largest_seqno, false, file_metadata.temperature,
kInvalidBlobFileNumber, oldest_ancester_time, current_time,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, f.unique_id);
f.unique_id);
// If incoming sequence number is higher, update local sequence number.
if (file_metadata.largest_seqno > versions_->LastSequence()) {

@ -651,7 +651,6 @@ class Repairer {
table->meta.temperature, table->meta.oldest_blob_file_number,
table->meta.oldest_ancester_time, table->meta.file_creation_time,
table->meta.file_checksum, table->meta.file_checksum_func_name,
table->meta.min_timestamp, table->meta.max_timestamp,
table->meta.unique_id);
}
assert(next_file_number_ > 0);

@ -72,8 +72,7 @@ class VersionBuilderTest : public testing::Test {
/* marked_for_compact */ false, Temperature::kUnknown,
oldest_blob_file_number, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
f->compensated_file_size = file_size;
f->num_entries = num_entries;
f->num_deletions = num_deletions;
@ -134,8 +133,7 @@ class VersionBuilderTest : public testing::Test {
GetInternalKey(largest), smallest_seqno, largest_seqno,
marked_for_compaction, Temperature::kUnknown, blob_file_number,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
}
void UpdateVersionStorageInfo(VersionStorageInfo* vstorage) {
@ -180,8 +178,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveTo) {
2, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.DeleteFile(3, 27U);
EnvOptions env_options;
@ -224,8 +221,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic) {
3, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.DeleteFile(0, 1U);
version_edit.DeleteFile(0, 88U);
@ -271,8 +267,7 @@ TEST_F(VersionBuilderTest, ApplyAndSaveToDynamic2) {
4, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.DeleteFile(0, 1U);
version_edit.DeleteFile(0, 88U);
version_edit.DeleteFile(4, 6U);
@ -308,32 +303,27 @@ TEST_F(VersionBuilderTest, ApplyMultipleAndSaveTo) {
2, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 676, 0, 100U, GetInternalKey("401"), GetInternalKey("450"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 636, 0, 100U, GetInternalKey("601"), GetInternalKey("650"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 616, 0, 100U, GetInternalKey("501"), GetInternalKey("550"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 606, 0, 100U, GetInternalKey("701"), GetInternalKey("750"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
EnvOptions env_options;
constexpr TableCache* table_cache = nullptr;
@ -372,32 +362,27 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
2, 666, 0, 100U, GetInternalKey("301"), GetInternalKey("350"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 676, 0, 100U, GetInternalKey("401"), GetInternalKey("450"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 636, 0, 100U, GetInternalKey("601"), GetInternalKey("650"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 616, 0, 100U, GetInternalKey("501"), GetInternalKey("550"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit.AddFile(
2, 606, 0, 100U, GetInternalKey("701"), GetInternalKey("750"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
ASSERT_OK(version_builder.Apply(&version_edit));
VersionEdit version_edit2;
@ -405,16 +390,14 @@ TEST_F(VersionBuilderTest, ApplyDeleteAndSaveTo) {
2, 808, 0, 100U, GetInternalKey("901"), GetInternalKey("950"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
version_edit2.DeleteFile(2, 616);
version_edit2.DeleteFile(2, 636);
version_edit.AddFile(
2, 806, 0, 100U, GetInternalKey("801"), GetInternalKey("850"), 200, 200,
false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
ASSERT_OK(version_builder.Apply(&version_edit2));
ASSERT_OK(version_builder.SaveTo(&new_vstorage));
@ -525,8 +508,7 @@ TEST_F(VersionBuilderTest, ApplyFileDeletionAndAddition) {
largest_seqno, marked_for_compaction, Temperature::kUnknown,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
ASSERT_OK(builder.Apply(&addition));
@ -570,13 +552,12 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyInBase) {
constexpr SequenceNumber largest_seqno = 1000;
constexpr bool marked_for_compaction = false;
edit.AddFile(new_level, file_number, path_id, file_size,
GetInternalKey(smallest), GetInternalKey(largest),
smallest_seqno, largest_seqno, marked_for_compaction,
Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
edit.AddFile(
new_level, file_number, path_id, file_size, GetInternalKey(smallest),
GetInternalKey(largest), smallest_seqno, largest_seqno,
marked_for_compaction, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
const Status s = builder.Apply(&edit);
ASSERT_TRUE(s.IsCorruption());
@ -612,8 +593,7 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) {
marked_for_compaction, Temperature::kUnknown,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
ASSERT_OK(builder.Apply(&edit));
@ -626,8 +606,7 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAlreadyApplied) {
GetInternalKey(largest), smallest_seqno, largest_seqno,
marked_for_compaction, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
const Status s = builder.Apply(&other_edit);
ASSERT_TRUE(s.IsCorruption());
@ -663,8 +642,7 @@ TEST_F(VersionBuilderTest, ApplyFileAdditionAndDeletion) {
GetInternalKey(largest), smallest_seqno, largest_seqno,
marked_for_compaction, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
ASSERT_OK(builder.Apply(&addition));
@ -1233,8 +1211,7 @@ TEST_F(VersionBuilderTest, SaveBlobFilesToConcurrentJobs) {
smallest_seqno, largest_seqno, marked_for_compaction,
Temperature::kUnknown, blob_file_number,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
checksum_value, checksum_method, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
checksum_value, checksum_method, kNullUniqueId64x2);
edit.AddBlobFile(blob_file_number, total_blob_count, total_blob_bytes,
checksum_method, checksum_value);
@ -1321,8 +1298,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
Temperature::kUnknown,
/* oldest_blob_file_number */ 16, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
edit.AddFile(/* level */ 1, /* file_number */ 700, /* path_id */ 0,
/* file_size */ 100, /* smallest */ GetInternalKey("801"),
@ -1331,8 +1307,7 @@ TEST_F(VersionBuilderTest, CheckConsistencyForBlobFiles) {
Temperature::kUnknown,
/* oldest_blob_file_number */ 1000, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
edit.AddBlobFile(/* blob_file_number */ 1000, /* total_blob_count */ 2000,
/* total_blob_bytes */ 200000,
/* checksum_method */ std::string(),
@ -1553,8 +1528,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
Temperature::kUnknown,
/* oldest_blob_file_number */ 1, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
// Add an SST that does not reference any blob files.
edit.AddFile(
@ -1564,8 +1538,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
/* largest_seqno */ 2200, /* marked_for_compaction */ false,
Temperature::kUnknown, kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
// Delete a file that references a blob file.
edit.DeleteFile(/* level */ 1, /* file_number */ 6);
@ -1587,8 +1560,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
Temperature::kUnknown,
/* oldest_blob_file_number */ 3, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
// Trivially move a file that does not reference any blob files.
edit.DeleteFile(/* level */ 1, /* file_number */ 13);
@ -1600,7 +1572,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kNullUniqueId64x2);
// Add one more SST file that references a blob file, then promptly
// delete it in a second version edit before the new version gets saved.
@ -1613,8 +1585,7 @@ TEST_F(VersionBuilderTest, MaintainLinkedSstsForBlobFiles) {
Temperature::kUnknown,
/* oldest_blob_file_number */ 5, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
VersionEdit edit2;

@ -192,16 +192,6 @@ bool VersionEdit::EncodeTo(std::string* dst) const {
PutVarint32(dst, NewFileCustomTag::kFileChecksumFuncName);
PutLengthPrefixedSlice(dst, Slice(f.file_checksum_func_name));
if (f.max_timestamp != kDisableUserTimestamp) {
if (f.min_timestamp.size() != f.max_timestamp.size()) {
assert(false);
return false;
}
PutVarint32(dst, NewFileCustomTag::kMinTimestamp);
PutLengthPrefixedSlice(dst, Slice(f.min_timestamp));
PutVarint32(dst, NewFileCustomTag::kMaxTimestamp);
PutLengthPrefixedSlice(dst, Slice(f.max_timestamp));
}
if (f.fd.GetPathId() != 0) {
PutVarint32(dst, NewFileCustomTag::kPathId);
char p = static_cast<char>(f.fd.GetPathId());
@ -339,10 +329,6 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
return "new-file4 custom field";
}
if (custom_tag == kTerminate) {
if (f.min_timestamp.size() != f.max_timestamp.size()) {
assert(false);
return "new-file4 custom field timestamp size mismatch error";
}
break;
}
if (!GetLengthPrefixedSlice(input, &field)) {
@ -403,12 +389,6 @@ const char* VersionEdit::DecodeNewFile4From(Slice* input) {
}
}
break;
case kMinTimestamp:
f.min_timestamp = field.ToString();
break;
case kMaxTimestamp:
f.max_timestamp = field.ToString();
break;
case kUniqueId:
if (!DecodeUniqueIdBytes(field.ToString(), &f.unique_id).ok()) {
f.unique_id = kNullUniqueId64x2;
@ -827,13 +807,6 @@ std::string VersionEdit::DebugString(bool hex_key) const {
r.append(" blob_file:");
AppendNumberTo(&r, f.oldest_blob_file_number);
}
if (f.min_timestamp != kDisableUserTimestamp) {
assert(f.max_timestamp != kDisableUserTimestamp);
r.append(" min_timestamp:");
r.append(Slice(f.min_timestamp).ToString(true));
r.append(" max_timestamp:");
r.append(Slice(f.max_timestamp).ToString(true));
}
r.append(" oldest_ancester_time:");
AppendNumberTo(&r, f.oldest_ancester_time);
r.append(" file_creation_time:");
@ -955,11 +928,6 @@ std::string VersionEdit::DebugJSON(int edit_num, bool hex_key) const {
jw << "FileSize" << f.fd.GetFileSize();
jw << "SmallestIKey" << f.smallest.DebugString(hex_key);
jw << "LargestIKey" << f.largest.DebugString(hex_key);
if (f.min_timestamp != kDisableUserTimestamp) {
assert(f.max_timestamp != kDisableUserTimestamp);
jw << "MinTimestamp" << Slice(f.min_timestamp).ToString(true);
jw << "MaxTimestamp" << Slice(f.max_timestamp).ToString(true);
}
jw << "OldestAncesterTime" << f.oldest_ancester_time;
jw << "FileCreationTime" << f.file_creation_time;
jw << "FileChecksum" << Slice(f.file_checksum).ToString(true);

@ -216,10 +216,6 @@ struct FileMetaData {
// File checksum function name
std::string file_checksum_func_name = kUnknownFileChecksumFuncName;
// Min (oldest) timestamp of keys in this file
std::string min_timestamp;
// Max (newest) timestamp of keys in this file
std::string max_timestamp;
// SST unique id
UniqueId64x2 unique_id{};
@ -234,7 +230,6 @@ struct FileMetaData {
uint64_t _oldest_ancester_time, uint64_t _file_creation_time,
const std::string& _file_checksum,
const std::string& _file_checksum_func_name,
std::string _min_timestamp, std::string _max_timestamp,
UniqueId64x2 _unique_id)
: fd(file, file_path_id, file_size, smallest_seq, largest_seq),
smallest(smallest_key),
@ -246,8 +241,6 @@ struct FileMetaData {
file_creation_time(_file_creation_time),
file_checksum(_file_checksum),
file_checksum_func_name(_file_checksum_func_name),
min_timestamp(std::move(_min_timestamp)),
max_timestamp(std::move(_max_timestamp)),
unique_id(std::move(_unique_id)) {
TEST_SYNC_POINT_CALLBACK("FileMetaData::FileMetaData", this);
}
@ -309,8 +302,7 @@ struct FileMetaData {
usage += sizeof(*this);
#endif // ROCKSDB_MALLOC_USABLE_SIZE
usage += smallest.size() + largest.size() + file_checksum.size() +
file_checksum_func_name.size() + min_timestamp.size() +
max_timestamp.size();
file_checksum_func_name.size();
return usage;
}
};
@ -435,8 +427,6 @@ class VersionEdit {
uint64_t oldest_ancester_time, uint64_t file_creation_time,
const std::string& file_checksum,
const std::string& file_checksum_func_name,
const std::string& min_timestamp,
const std::string& max_timestamp,
const UniqueId64x2& unique_id) {
assert(smallest_seqno <= largest_seqno);
new_files_.emplace_back(
@ -445,7 +435,7 @@ class VersionEdit {
smallest_seqno, largest_seqno, marked_for_compaction,
temperature, oldest_blob_file_number, oldest_ancester_time,
file_creation_time, file_checksum, file_checksum_func_name,
min_timestamp, max_timestamp, unique_id));
unique_id));
if (!HasLastSequence() || largest_seqno > GetLastSequence()) {
SetLastSequence(largest_seqno);
}

@ -43,8 +43,8 @@ TEST_F(VersionEditTest, EncodeDecode) {
InternalKey("foo", kBig + 500 + i, kTypeValue),
InternalKey("zoo", kBig + 600 + i, kTypeDeletion),
kBig + 500 + i, kBig + 600 + i, false, Temperature::kUnknown,
kInvalidBlobFileNumber, 888, 678, "234", "crc32c", "123",
"345", kNullUniqueId64x2);
kInvalidBlobFileNumber, 888, 678, "234", "crc32c",
kNullUniqueId64x2);
edit.DeleteFile(4, kBig + 700 + i);
}
@ -63,24 +63,24 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "123", "234",
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kNullUniqueId64x2);
edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
kBig + 601, false, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "345", "543",
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kNullUniqueId64x2);
edit.AddFile(5, 302, 0, 100, InternalKey("foo", kBig + 502, kTypeValue),
InternalKey("zoo", kBig + 602, kTypeDeletion), kBig + 502,
kBig + 602, true, Temperature::kUnknown, kInvalidBlobFileNumber,
666, 888, kUnknownFileChecksum, kUnknownFileChecksumFuncName,
"456", "567", kNullUniqueId64x2);
kNullUniqueId64x2);
edit.AddFile(5, 303, 0, 100, InternalKey("foo", kBig + 503, kTypeBlobIndex),
InternalKey("zoo", kBig + 603, kTypeBlobIndex), kBig + 503,
kBig + 603, true, Temperature::kUnknown, 1001,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "678", "789",
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kNullUniqueId64x2);
edit.DeleteFile(4, 700);
@ -112,14 +112,6 @@ TEST_F(VersionEditTest, EncodeDecodeNewFile4) {
ASSERT_EQ(kInvalidBlobFileNumber,
new_files[2].second.oldest_blob_file_number);
ASSERT_EQ(1001, new_files[3].second.oldest_blob_file_number);
ASSERT_EQ("123", new_files[0].second.min_timestamp);
ASSERT_EQ("234", new_files[0].second.max_timestamp);
ASSERT_EQ("345", new_files[1].second.min_timestamp);
ASSERT_EQ("543", new_files[1].second.max_timestamp);
ASSERT_EQ("456", new_files[2].second.min_timestamp);
ASSERT_EQ("567", new_files[2].second.max_timestamp);
ASSERT_EQ("678", new_files[3].second.min_timestamp);
ASSERT_EQ("789", new_files[3].second.max_timestamp);
}
TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
@ -129,13 +121,12 @@ TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
InternalKey("zoo", kBig + 600, kTypeDeletion), kBig + 500,
kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName, "123", "234",
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kNullUniqueId64x2);
edit.AddFile(4, 301, 3, 100, InternalKey("foo", kBig + 501, kTypeValue),
InternalKey("zoo", kBig + 601, kTypeDeletion), kBig + 501,
kBig + 601, false, Temperature::kUnknown, kInvalidBlobFileNumber,
686, 868, "234", "crc32c", kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
686, 868, "234", "crc32c", kNullUniqueId64x2);
edit.DeleteFile(4, 700);
edit.SetComparatorName("foo");
@ -174,10 +165,6 @@ TEST_F(VersionEditTest, ForwardCompatibleNewFile4) {
ASSERT_EQ(3u, new_files[0].second.fd.GetPathId());
ASSERT_EQ(3u, new_files[1].second.fd.GetPathId());
ASSERT_EQ(1u, parsed.GetDeletedFiles().size());
ASSERT_EQ("123", new_files[0].second.min_timestamp);
ASSERT_EQ("234", new_files[0].second.max_timestamp);
ASSERT_EQ(kDisableUserTimestamp, new_files[1].second.min_timestamp);
ASSERT_EQ(kDisableUserTimestamp, new_files[1].second.max_timestamp);
}
TEST_F(VersionEditTest, NewFile4NotSupportedField) {
@ -188,7 +175,7 @@ TEST_F(VersionEditTest, NewFile4NotSupportedField) {
kBig + 600, true, Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kNullUniqueId64x2);
edit.SetComparatorName("foo");
edit.SetLogNumber(kBig + 100);
@ -219,7 +206,7 @@ TEST_F(VersionEditTest, EncodeEmptyFile) {
Temperature::kUnknown, kInvalidBlobFileNumber,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kNullUniqueId64x2);
std::string buffer;
ASSERT_TRUE(!edit.EncodeTo(&buffer));
}

@ -5550,8 +5550,7 @@ Status VersionSet::WriteCurrentStateToManifest(
f->marked_for_compaction, f->temperature,
f->oldest_blob_file_number, f->oldest_ancester_time,
f->file_creation_time, f->file_checksum,
f->file_checksum_func_name, f->min_timestamp,
f->max_timestamp, f->unique_id);
f->file_checksum_func_name, f->unique_id);
}
}

@ -50,8 +50,7 @@ class GenerateLevelFilesBriefTest : public testing::Test {
largest_seq, /* marked_for_compact */ false, Temperature::kUnknown,
kInvalidBlobFileNumber, kUnknownOldestAncesterTime,
kUnknownFileCreationTime, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
files_.push_back(f);
}
@ -159,8 +158,7 @@ class VersionStorageInfoTestBase : public testing::Test {
/* largest_seq */ 0, /* marked_for_compact */ false,
Temperature::kUnknown, oldest_blob_file_number,
kUnknownOldestAncesterTime, kUnknownFileCreationTime,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
f->compensated_file_size = file_size;
vstorage_.AddFile(level, f);
}
@ -3233,11 +3231,10 @@ class VersionSetTestMissingFiles : public VersionSetTestBase,
s = fs_->GetFileSize(fname, IOOptions(), &file_size, nullptr);
ASSERT_OK(s);
ASSERT_NE(0, file_size);
file_metas->emplace_back(
file_num, /*file_path_id=*/0, file_size, ikey, ikey, 0, 0, false,
Temperature::kUnknown, 0, 0, 0, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kDisableUserTimestamp,
kDisableUserTimestamp, kNullUniqueId64x2);
file_metas->emplace_back(file_num, /*file_path_id=*/0, file_size, ikey,
ikey, 0, 0, false, Temperature::kUnknown, 0, 0,
0, kUnknownFileChecksum,
kUnknownFileChecksumFuncName, kNullUniqueId64x2);
}
}
@ -3292,8 +3289,7 @@ TEST_F(VersionSetTestMissingFiles, ManifestFarBehindSst) {
FileMetaData meta = FileMetaData(
file_num, /*file_path_id=*/0, /*file_size=*/12, smallest_ikey,
largest_ikey, 0, 0, false, Temperature::kUnknown, 0, 0, 0,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
added_files.emplace_back(0, meta);
}
WriteFileAdditionAndDeletionToManifest(
@ -3348,8 +3344,7 @@ TEST_F(VersionSetTestMissingFiles, ManifestAheadofSst) {
FileMetaData meta = FileMetaData(
file_num, /*file_path_id=*/0, /*file_size=*/12, smallest_ikey,
largest_ikey, 0, 0, false, Temperature::kUnknown, 0, 0, 0,
kUnknownFileChecksum, kUnknownFileChecksumFuncName,
kDisableUserTimestamp, kDisableUserTimestamp, kNullUniqueId64x2);
kUnknownFileChecksum, kUnknownFileChecksumFuncName, kNullUniqueId64x2);
added_files.emplace_back(0, meta);
}
WriteFileAdditionAndDeletionToManifest(

Loading…
Cancel
Save