Use madvise() for mmaped file advise (#10170)

Summary:
A recent PR https://github.com/facebook/rocksdb/pull/10142 enabled fadvise for mmaped file. However, we were told that it might not take effective and madvise() should be used.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/10170

Test Plan:
Run existing tests
Run a benchmark using mmap with advise random and see I/O size is indeed small.

Reviewed By: anand1976

Differential Revision: D37158582

fbshipit-source-id: 8b3a74f0e89d2e16aac78ee4124c05841d4135c3

env/io_posix.cc

@@ -88,22 +88,17 @@ int Fadvise(int fd, off_t offset, size_t len, int advice) {
 #endif
 }
 
-int FadviseForHint(int fd, FSRandomAccessFile::AccessPattern pattern) {
+// A wrapper for fadvise, if the platform doesn't support fadvise,
-  switch (pattern) {
+// it will simply return 0.
-    case FSRandomAccessFile::AccessPattern::kNormal:
+int Madvise(void* addr, size_t len, int advice) {
-      return Fadvise(fd, 0, 0, POSIX_FADV_NORMAL);
+#ifdef OS_LINUX
-    case FSRandomAccessFile::AccessPattern::kRandom:
+  return posix_madvise(addr, len, advice);
-      return Fadvise(fd, 0, 0, POSIX_FADV_RANDOM);
+#else
-    case FSRandomAccessFile::AccessPattern::kSequential:
+  (void)addr;
-      return Fadvise(fd, 0, 0, POSIX_FADV_SEQUENTIAL);
+  (void)len;
-    case FSRandomAccessFile::AccessPattern::kWillNeed:
+  (void)advice;
-      return Fadvise(fd, 0, 0, POSIX_FADV_WILLNEED);
+  return 0;  // simply do nothing.
-    case FSRandomAccessFile::AccessPattern::kWontNeed:
+#endif
-      return Fadvise(fd, 0, 0, POSIX_FADV_DONTNEED);
-    default:
-      assert(false);
-      return 1;
-  }
 }
 
 namespace {
@@ -839,7 +834,26 @@ void PosixRandomAccessFile::Hint(AccessPattern pattern) {
   if (use_direct_io()) {
     return;
   }
-  FadviseForHint(fd_, pattern);
+  switch (pattern) {
+    case kNormal:
+      Fadvise(fd_, 0, 0, POSIX_FADV_NORMAL);
+      break;
+    case kRandom:
+      Fadvise(fd_, 0, 0, POSIX_FADV_RANDOM);
+      break;
+    case kSequential:
+      Fadvise(fd_, 0, 0, POSIX_FADV_SEQUENTIAL);
+      break;
+    case kWillNeed:
+      Fadvise(fd_, 0, 0, POSIX_FADV_WILLNEED);
+      break;
+    case kWontNeed:
+      Fadvise(fd_, 0, 0, POSIX_FADV_DONTNEED);
+      break;
+    default:
+      assert(false);
+      break;
+  }
 }
 
 IOStatus PosixRandomAccessFile::InvalidateCache(size_t offset, size_t length) {
@@ -982,7 +996,26 @@ IOStatus PosixMmapReadableFile::Read(uint64_t offset, size_t n,
 }
 
 void PosixMmapReadableFile::Hint(AccessPattern pattern) {
-  FadviseForHint(fd_, pattern);
+  switch (pattern) {
+    case kNormal:
+      Madvise(mmapped_region_, length_, POSIX_MADV_NORMAL);
+      break;
+    case kRandom:
+      Madvise(mmapped_region_, length_, POSIX_MADV_RANDOM);
+      break;
+    case kSequential:
+      Madvise(mmapped_region_, length_, POSIX_MADV_SEQUENTIAL);
+      break;
+    case kWillNeed:
+      Madvise(mmapped_region_, length_, POSIX_MADV_WILLNEED);
+      break;
+    case kWontNeed:
+      Madvise(mmapped_region_, length_, POSIX_MADV_DONTNEED);
+      break;
+    default:
+      assert(false);
+      break;
+  }
 }
 
 IOStatus PosixMmapReadableFile::InvalidateCache(size_t offset, size_t length) {