From a89740fbc6ce84331801b93158a29d9b515e0dbd Mon Sep 17 00:00:00 2001
From: Jay Zhuang <zjay@fb.com>
Date: Tue, 20 Apr 2021 08:41:32 -0700
Subject: [PATCH] Fix unittest no space issue (#8204)

Summary:
Unittest reports no space from time to time, which can be reproduced on a small memory machine with SHM. It's caused by large WAL files generated during the test, which is preallocated, but didn't truncate during close(). Adding the missing APIs to set preallocation.
It added arm test as nightly build, as the test runs more than 1 hour.

Pull Request resolved: https://github.com/facebook/rocksdb/pull/8204

Test Plan: test on small memory arm machine

Reviewed By: mrambacher

Differential Revision: D27873145

Pulled By: jay-zhuang

fbshipit-source-id: f797c429d6bc13cbcc673bc03fcc72adda55f506
---
 .circleci/config.yml             | 12 ++++++++++++
 db/db_test_util.h                | 12 +++++++++---
 env/env_encryption.cc            | 11 +++++++++++
 include/rocksdb/env_encryption.h |  5 +++++
 4 files changed, 37 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 15fd6ef4d..ec6d63a5b 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -572,6 +572,16 @@ jobs:
             gtest-parallel $(</tmp/test_list) --output_dir=/tmp | cat  # pipe to cat to continuously output status on circleci UI. Otherwise, no status will be printed while the job is running.
       - post-steps
 
+  build-linux-arm-test-full:
+    machine:
+      image: ubuntu-2004:202101-01
+    resource_class: arm.large
+    steps:
+      - pre-steps
+      - install-gflags
+      - run: make V=1 J=4 -j4 check | .circleci/cat_ignore_eagain
+      - post-steps
+
   build-linux-arm:
     machine:
       image: ubuntu-2004:202101-01
@@ -612,6 +622,7 @@ jobs:
             cd build
             cmake -DJNI=1 -DCMAKE_BUILD_TYPE=Release -DWITH_GFLAGS=0 ..
             make -j4 rocksdb rocksdbjni
+      - post-steps
 
   build-format-compatible:
     machine:
@@ -746,3 +757,4 @@ workflows:
                 - master
     jobs:
       - build-format-compatible
+      - build-linux-arm-test-full
diff --git a/db/db_test_util.h b/db/db_test_util.h
index 8dc0e3a33..cd34bd21e 100644
--- a/db/db_test_util.h
+++ b/db/db_test_util.h
@@ -378,14 +378,20 @@ class SpecialEnv : public EnvWrapper {
         return Append(data);
       }
       Status Truncate(uint64_t size) override { return base_->Truncate(size); }
+      void PrepareWrite(size_t offset, size_t len) override {
+        base_->PrepareWrite(offset, len);
+      }
+      void SetPreallocationBlockSize(size_t size) override {
+        base_->SetPreallocationBlockSize(size);
+      }
       Status Close() override {
 // SyncPoint is not supported in Released Windows Mode.
 #if !(defined NDEBUG) || !defined(OS_WIN)
         // Check preallocation size
-        // preallocation size is never passed to base file.
-        size_t preallocation_size = preallocation_block_size();
+        size_t block_size, last_allocated_block;
+        base_->GetPreallocationStatus(&block_size, &last_allocated_block);
         TEST_SYNC_POINT_CALLBACK("DBTestWalFile.GetPreallocationStatus",
-                                 &preallocation_size);
+                                 &block_size);
 #endif  // !(defined NDEBUG) || !defined(OS_WIN)
 
         return base_->Close();
diff --git a/env/env_encryption.cc b/env/env_encryption.cc
index d34125314..a5670ad78 100644
--- a/env/env_encryption.cc
+++ b/env/env_encryption.cc
@@ -362,6 +362,17 @@ void EncryptedWritableFile::PrepareWrite(size_t offset, size_t len,
   file_->PrepareWrite(offset + prefixLength_, len, options, dbg);
 }
 
+void EncryptedWritableFile::SetPreallocationBlockSize(size_t size) {
+  // the size here doesn't need to include prefixLength_, as it's a
+  // configuration will be use for `PrepareWrite()`.
+  file_->SetPreallocationBlockSize(size);
+}
+
+void EncryptedWritableFile::GetPreallocationStatus(
+    size_t* block_size, size_t* last_allocated_block) {
+  file_->GetPreallocationStatus(block_size, last_allocated_block);
+}
+
 // Pre-allocates space for a file.
 IOStatus EncryptedWritableFile::Allocate(uint64_t offset, uint64_t len,
                                          const IOOptions& options,
diff --git a/include/rocksdb/env_encryption.h b/include/rocksdb/env_encryption.h
index 8ac5da973..7a76ec867 100644
--- a/include/rocksdb/env_encryption.h
+++ b/include/rocksdb/env_encryption.h
@@ -366,6 +366,11 @@ class EncryptedWritableFile : public FSWritableFile {
   void PrepareWrite(size_t offset, size_t len, const IOOptions& options,
                     IODebugContext* dbg) override;
 
+  void SetPreallocationBlockSize(size_t size) override;
+
+  void GetPreallocationStatus(size_t* block_size,
+                              size_t* last_allocated_block) override;
+
   // Pre-allocates space for a file.
   IOStatus Allocate(uint64_t offset, uint64_t len, const IOOptions& options,
                     IODebugContext* dbg) override;