Add more ops to: db_bench -report_file_operations (#8448)

Summary:
Hello and thanks for RocksDB,

Here is a PR to add file deletes, renames and ```Flush()```, ```Sync()```, ```Fsync()``` and ```Close()``` to file ops report.

The reason is to help tune RocksDB options when using an env/filesystem with high latencies for file level ("metadata") operations, typically seen during ```DB::Open``` (```db_bench -num 0``` also see https://github.com/facebook/rocksdb/pull/7203 where IOTracing does not trace ```DB::Open```).

Before:
```
> db_bench -benchmarks updaterandom -num 0 -report_file_operations true
...
Entries:    0
...
Num files opened: 12
Num Read(): 6
Num Append(): 8
Num bytes read: 6216
Num bytes written: 6289
```
After:
```
> db_bench -benchmarks updaterandom -num 0 -report_file_operations true
...
Entries:    0
...
Num files opened: 12
Num files deleted: 3
Num files renamed: 4
Num Flush(): 10
Num Sync(): 5
Num Fsync(): 1
Num Close(): 2
Num Read(): 6
Num Append(): 8
Num bytes read: 6216
Num bytes written: 6289
```

Before:
```
> db_bench -benchmarks updaterandom -report_file_operations true
...
Entries:    1000000
...
Num files opened: 18
Num Read(): 396339
Num Append(): 1000058
Num bytes read: 892030224
Num bytes written: 187569238
```
After:
```
> db_bench -benchmarks updaterandom -report_file_operations true
...
Entries:    1000000
...
Num files opened: 18
Num files deleted: 5
Num files renamed: 4
Num Flush(): 1000068
Num Sync(): 9
Num Fsync(): 1
Num Close(): 6
Num Read(): 396339
Num Append(): 1000058
Num bytes read: 892030224
Num bytes written: 187569238
```

Another example showing how using ```DB::OpenForReadOnly``` reduces file operations compared to ```((Optimistic)Transaction)DB::Open```:

```
> db_bench -benchmarks updaterandom -num 1
> db_bench -benchmarks updaterandom -num 0 -use_existing_db true -readonly true -report_file_operations true
...
Entries:    0
...
Num files opened: 8
Num files deleted: 0
Num files renamed: 0
Num Flush(): 0
Num Sync(): 0
Num Fsync(): 0
Num Close(): 0
Num Read(): 13
Num Append(): 0
Num bytes read: 374
Num bytes written: 0
```

```
> db_bench -benchmarks updaterandom -num 1
> db_bench -benchmarks updaterandom -num 0 -use_existing_db true -report_file_operations true
...
Entries:    0
...
Num files opened: 14
Num files deleted: 3
Num files renamed: 4
Num Flush(): 14
Num Sync(): 5
Num Fsync(): 1
Num Close(): 3
Num Read(): 11
Num Append(): 10
Num bytes read: 7291
Num bytes written: 7357
```

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8448

Reviewed By: anand1976

Differential Revision: D29333818

Pulled By: zhichao-cao

fbshipit-source-id: a06a8c87f799806462319115195b3e94faf5f542
main
Peter (Stig) Edwards 3 years ago committed by Facebook GitHub Bot
parent d44ef2ed4d
commit 75741eb0ce
  1. 75
      tools/db_bench_tool.cc

@ -1455,6 +1455,12 @@ namespace ROCKSDB_NAMESPACE {
namespace {
struct ReportFileOpCounters {
std::atomic<int> open_counter_;
std::atomic<int> delete_counter_;
std::atomic<int> rename_counter_;
std::atomic<int> flush_counter_;
std::atomic<int> sync_counter_;
std::atomic<int> fsync_counter_;
std::atomic<int> close_counter_;
std::atomic<int> read_counter_;
std::atomic<int> append_counter_;
std::atomic<uint64_t> bytes_read_;
@ -1468,6 +1474,12 @@ class ReportFileOpEnv : public EnvWrapper {
void reset() {
counters_.open_counter_ = 0;
counters_.delete_counter_ = 0;
counters_.rename_counter_ = 0;
counters_.flush_counter_ = 0;
counters_.sync_counter_ = 0;
counters_.fsync_counter_ = 0;
counters_.close_counter_ = 0;
counters_.read_counter_ = 0;
counters_.append_counter_ = 0;
counters_.bytes_read_ = 0;
@ -1506,6 +1518,22 @@ class ReportFileOpEnv : public EnvWrapper {
return s;
}
Status DeleteFile(const std::string& fname) override {
Status s = target()->DeleteFile(fname);
if (s.ok()) {
counters()->delete_counter_.fetch_add(1, std::memory_order_relaxed);
}
return s;
}
Status RenameFile(const std::string& s, const std::string& t) override {
Status st = target()->RenameFile(s, t);
if (st.ok()) {
counters()->rename_counter_.fetch_add(1, std::memory_order_relaxed);
}
return st;
}
Status NewRandomAccessFile(const std::string& f,
std::unique_ptr<RandomAccessFile>* r,
const EnvOptions& soptions) override {
@ -1562,10 +1590,37 @@ class ReportFileOpEnv : public EnvWrapper {
return Append(data);
}
Status Truncate(uint64_t size) override { return target_->Truncate(size); }
Status Close() override { return target_->Close(); }
Status Flush() override { return target_->Flush(); }
Status Sync() override { return target_->Sync(); }
Status Truncate(uint64_t size) override {
return target_->Truncate(size);
}
Status Close() override {
Status s = target_->Close();
if (s.ok()) {
counters_->close_counter_.fetch_add(1, std::memory_order_relaxed);
}
return s;
}
Status Flush() override {
Status s = target_->Flush();
if (s.ok()) {
counters_->flush_counter_.fetch_add(1, std::memory_order_relaxed);
}
return s;
}
Status Sync() override {
Status s = target_->Sync();
if (s.ok()) {
counters_->sync_counter_.fetch_add(1, std::memory_order_relaxed);
}
return s;
}
Status Fsync() override {
Status s = target_->Fsync();
if (s.ok()) {
counters_->fsync_counter_.fetch_add(1, std::memory_order_relaxed);
}
return s;
}
};
Status s = target()->NewWritableFile(f, r, soptions);
@ -2242,6 +2297,18 @@ class Stats {
ReportFileOpCounters* counters = env->counters();
fprintf(stdout, "Num files opened: %d\n",
counters->open_counter_.load(std::memory_order_relaxed));
fprintf(stdout, "Num files deleted: %d\n",
counters->delete_counter_.load(std::memory_order_relaxed));
fprintf(stdout, "Num files renamed: %d\n",
counters->rename_counter_.load(std::memory_order_relaxed));
fprintf(stdout, "Num Flush(): %d\n",
counters->flush_counter_.load(std::memory_order_relaxed));
fprintf(stdout, "Num Sync(): %d\n",
counters->sync_counter_.load(std::memory_order_relaxed));
fprintf(stdout, "Num Fsync(): %d\n",
counters->fsync_counter_.load(std::memory_order_relaxed));
fprintf(stdout, "Num Close(): %d\n",
counters->close_counter_.load(std::memory_order_relaxed));
fprintf(stdout, "Num Read(): %d\n",
counters->read_counter_.load(std::memory_order_relaxed));
fprintf(stdout, "Num Append(): %d\n",

Loading…
Cancel
Save