Direct IO fix for Mac

Summary:
O_DIRECT is not available in Mac as a flag for open. The fix is to make
use of fctl after the file is opened

Test Plan: Run the tests on mac and Linux

Reviewers: sdong

Subscribers: andrewkr, dhruba, leveldb

Differential Revision: https://reviews.facebook.net/D58665

util/env_posix.cc

@@ -153,12 +153,22 @@ class PosixEnv : public Env {
       *result = nullptr;
       return IOError(fname, errno);
     } else if (options.use_direct_reads && !options.use_mmap_writes) {
+#ifdef OS_MACOSX
+      int flags = O_RDONLY;
+#else
       int flags = O_RDONLY | O_DIRECT;
       TEST_SYNC_POINT_CALLBACK("NewSequentialFile:O_DIRECT", &flags);
+#endif
       int fd = open(fname.c_str(), flags, 0644);
       if (fd < 0) {
         return IOError(fname, errno);
       }
+#ifdef OS_MACOSX
+      if (fcntl(fd, F_NOCACHE, 1) == -1) {
+        close(fd);
+        return IOError(fname, errno);
+      }
+#endif
       std::unique_ptr<PosixDirectIOSequentialFile> file(
           new PosixDirectIOSequentialFile(fname, fd));
       *result = std::move(file);
@@ -201,8 +211,12 @@ class PosixEnv : public Env {
       }
       close(fd);
     } else if (options.use_direct_reads) {
+#ifdef OS_MACOSX
+      int flags = O_RDONLY;
+#else
       int flags = O_RDONLY | O_DIRECT;
       TEST_SYNC_POINT_CALLBACK("NewRandomAccessFile:O_DIRECT", &flags);
+#endif
       fd = open(fname.c_str(), flags, 0644);
       if (fd < 0) {
         s = IOError(fname, errno);
@@ -211,6 +225,12 @@ class PosixEnv : public Env {
             new PosixDirectIORandomAccessFile(fname, fd));
         *result = std::move(file);
         s = Status::OK();
+#ifdef OS_MACOSX
+        if (fcntl(fd, F_NOCACHE, 1) == -1) {
+          close(fd);
+          s = IOError(fname, errno);
+        }
+#endif
       }
     } else {
       result->reset(new PosixRandomAccessFile(fname, fd, options));
@@ -245,7 +265,11 @@ class PosixEnv : public Env {
       if (options.use_mmap_writes && !forceMmapOff) {
         result->reset(new PosixMmapFile(fname, fd, page_size_, options));
       } else if (options.use_direct_writes) {
+#ifdef OS_MACOSX
+        int flags = O_WRONLY | O_APPEND | O_TRUNC | O_CREAT;
+#else
         int flags = O_WRONLY | O_APPEND | O_TRUNC | O_CREAT | O_DIRECT;
+#endif
         TEST_SYNC_POINT_CALLBACK("NewWritableFile:O_DIRECT", &flags);
         fd = open(fname.c_str(), flags, 0644);
         if (fd < 0) {
@@ -255,6 +279,12 @@ class PosixEnv : public Env {
               new PosixDirectIOWritableFile(fname, fd));
           *result = std::move(file);
           s = Status::OK();
+#ifdef OS_MACOSX
+          if (fcntl(fd, F_NOCACHE, 1) == -1) {
+            close(fd);
+            s = IOError(fname, errno);
+          }
+#endif
         }
       } else {
         // disable mmap writes

util/env_test.cc

@@ -794,6 +794,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
     // Create file.
     {
       unique_ptr<WritableFile> wfile;
+#ifndef OS_MACOSX
       if (soptions.use_direct_writes) {
         rocksdb::SyncPoint::GetInstance()->SetCallBack(
             "NewWritableFile:O_DIRECT", [&](void* arg) {
@@ -801,7 +802,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
               *val &= ~O_DIRECT;
             });
       }
-
+#endif
       ASSERT_OK(env_->NewWritableFile(fname, &wfile, soptions));
       ASSERT_OK(wfile.get()->Append(slice));
       ASSERT_OK(wfile.get()->InvalidateCache(0, 0));
@@ -813,6 +814,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
       unique_ptr<RandomAccessFile> file;
       char scratch[kSectorSize];
       Slice result;
+#ifndef OS_MACOSX
       if (soptions.use_direct_reads) {
         rocksdb::SyncPoint::GetInstance()->SetCallBack(
             "NewRandomAccessFile:O_DIRECT", [&](void* arg) {
@@ -820,7 +822,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
               *val &= ~O_DIRECT;
             });
       }
-
+#endif
       ASSERT_OK(env_->NewRandomAccessFile(fname, &file, soptions));
       ASSERT_OK(file.get()->Read(0, kSectorSize, &result, scratch));
       ASSERT_EQ(memcmp(scratch, data.get(), kSectorSize), 0);
@@ -833,6 +835,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
       unique_ptr<SequentialFile> file;
       char scratch[kSectorSize];
       Slice result;
+#ifndef OS_MACOSX
       if (soptions.use_direct_reads) {
         rocksdb::SyncPoint::GetInstance()->SetCallBack(
             "NewSequentialFile:O_DIRECT", [&](void* arg) {
@@ -840,7 +843,7 @@ TEST_P(EnvPosixTestWithParam, InvalidateCache) {
               *val &= ~O_DIRECT;
             });
       }
-
+#endif
       ASSERT_OK(env_->NewSequentialFile(fname, &file, soptions));
       ASSERT_OK(file.get()->Read(kSectorSize, &result, scratch));
       ASSERT_EQ(memcmp(scratch, data.get(), kSectorSize), 0);
@@ -978,6 +981,7 @@ TEST_P(EnvPosixTestWithParam, Preallocation) {
     unique_ptr<WritableFile> srcfile;
     EnvOptions soptions;
     soptions.use_direct_reads = soptions.use_direct_writes = directio;
+#ifndef OS_MACOSX
     if (soptions.use_direct_writes) {
       rocksdb::SyncPoint::GetInstance()->SetCallBack(
           "NewWritableFile:O_DIRECT", [&](void* arg) {
@@ -985,7 +989,7 @@ TEST_P(EnvPosixTestWithParam, Preallocation) {
             *val &= ~O_DIRECT;
           });
     }
-
+#endif
     ASSERT_OK(env_->NewWritableFile(src, &srcfile, soptions));
     srcfile->SetPreallocationBlockSize(1024 * 1024);
 
@@ -1041,6 +1045,7 @@ TEST_P(EnvPosixTestWithParam, ConsistentChildrenAttributes) {
       oss << test::TmpDir(env_) << "/testfile_" << i;
       const std::string path = oss.str();
       unique_ptr<WritableFile> file;
+#ifndef OS_MACOSX
       if (soptions.use_direct_writes) {
         rocksdb::SyncPoint::GetInstance()->SetCallBack(
             "NewWritableFile:O_DIRECT", [&](void* arg) {
@@ -1048,7 +1053,7 @@ TEST_P(EnvPosixTestWithParam, ConsistentChildrenAttributes) {
               *val &= ~O_DIRECT;
             });
       }
-
+#endif
       ASSERT_OK(env_->NewWritableFile(path, &file, soptions));
       auto buf_ptr = NewAligned(data.size(), 'T');
       Slice buf(buf_ptr.get(), data.size());